Total coverage: 136221 (7%)of 1974400 

































  310 









   66 

   67 








  147 







  147 
   17 










   68 









   68 
   68 
   67 
   68 



































   68 





   58 

   58 



   58 




   58 



   22 












   68 





   68 

















   68 
   67 

   68 



   67 
















   68 






   21 
   67 







   68 


   68 
















  310 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
// SPDX-License-Identifier: GPL-2.0-or-later
/* SCTP kernel implementation
 * Copyright (c) 1999-2000 Cisco, Inc.
 * Copyright (c) 1999-2001 Motorola, Inc.
 * Copyright (c) 2002 International Business Machines, Corp.
 *
 * This file is part of the SCTP kernel implementation
 *
 * These functions are the methods for accessing the SCTP inqueue.
 *
 * An SCTP inqueue is a queue into which you push SCTP packets
 * (which might be bundles or fragments of chunks) and out of which you
 * pop SCTP whole chunks.
 *
 * Please send any bug reports or fixes you make to the
 * email address(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *    La Monte H.P. Yarroll <piggy@acm.org>
 *    Karl Knutson <karl@athena.chicago.il.us>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <net/sctp/sctp.h>
#include <net/sctp/sm.h>
#include <linux/interrupt.h>
#include <linux/slab.h>

/* Initialize an SCTP inqueue.  */
void sctp_inq_init(struct sctp_inq *queue)
{
        INIT_LIST_HEAD(&queue->in_chunk_list);
        queue->in_progress = NULL;

        /* Create a task for delivering data.  */
        INIT_WORK(&queue->immediate, NULL);
}

/* Properly release the chunk which is being worked on. */
static inline void sctp_inq_chunk_free(struct sctp_chunk *chunk)
{
        if (chunk->head_skb)
                chunk->skb = chunk->head_skb;
        sctp_chunk_free(chunk);
}

/* Release the memory associated with an SCTP inqueue.  */
void sctp_inq_free(struct sctp_inq *queue)
{
        struct sctp_chunk *chunk, *tmp;

        /* Empty the queue.  */
        list_for_each_entry_safe(chunk, tmp, &queue->in_chunk_list, list) {
                list_del_init(&chunk->list);
                sctp_chunk_free(chunk);
        }

        /* If there is a packet which is currently being worked on,
         * free it as well.
         */
        if (queue->in_progress) {
                sctp_inq_chunk_free(queue->in_progress);
                queue->in_progress = NULL;
        }
}

/* Put a new packet in an SCTP inqueue.
 * We assume that packet->sctp_hdr is set and in host byte order.
 */
void sctp_inq_push(struct sctp_inq *q, struct sctp_chunk *chunk)
{
        /* Directly call the packet handling routine. */
        if (chunk->rcvr->dead) {
                sctp_chunk_free(chunk);
                return;
        }

        /* We are now calling this either from the soft interrupt
         * or from the backlog processing.
         * Eventually, we should clean up inqueue to not rely
         * on the BH related data structures.
         */
        list_add_tail(&chunk->list, &q->in_chunk_list);
        if (chunk->asoc)
                chunk->asoc->stats.ipackets++;
        q->immediate.func(&q->immediate);
}

/* Peek at the next chunk on the inqeue. */
struct sctp_chunkhdr *sctp_inq_peek(struct sctp_inq *queue)
{
        struct sctp_chunk *chunk;
        struct sctp_chunkhdr *ch = NULL;

        chunk = queue->in_progress;
        /* If there is no more chunks in this packet, say so */
        if (chunk->singleton ||
            chunk->end_of_packet ||
            chunk->pdiscard)
                    return NULL;

        ch = (struct sctp_chunkhdr *)chunk->chunk_end;

        return ch;
}


/* Extract a chunk from an SCTP inqueue.
 *
 * WARNING:  If you need to put the chunk on another queue, you need to
 * make a shallow copy (clone) of it.
 */
struct sctp_chunk *sctp_inq_pop(struct sctp_inq *queue)
{
        struct sctp_chunk *chunk;
        struct sctp_chunkhdr *ch = NULL;

        /* The assumption is that we are safe to process the chunks
         * at this time.
         */

        chunk = queue->in_progress;
        if (chunk) {
                /* There is a packet that we have been working on.
                 * Any post processing work to do before we move on?
                 */
                if (chunk->singleton ||
                    chunk->end_of_packet ||
                    chunk->pdiscard) {
                        if (chunk->head_skb == chunk->skb) {
                                chunk->skb = skb_shinfo(chunk->skb)->frag_list;
                                goto new_skb;
                        }
                        if (chunk->skb->next) {
                                chunk->skb = chunk->skb->next;
                                goto new_skb;
                        }

                        sctp_inq_chunk_free(chunk);
                        chunk = queue->in_progress = NULL;
                } else {
                        /* Nothing to do. Next chunk in the packet, please. */
                        ch = (struct sctp_chunkhdr *)chunk->chunk_end;
                        /* Force chunk->skb->data to chunk->chunk_end.  */
                        skb_pull(chunk->skb, chunk->chunk_end - chunk->skb->data);
                        /* We are guaranteed to pull a SCTP header. */
                }
        }

        /* Do we need to take the next packet out of the queue to process? */
        if (!chunk) {
                struct list_head *entry;

next_chunk:
                /* Is the queue empty?  */
                entry = sctp_list_dequeue(&queue->in_chunk_list);
                if (!entry)
                        return NULL;

                chunk = list_entry(entry, struct sctp_chunk, list);

                if (skb_is_gso(chunk->skb) && skb_is_gso_sctp(chunk->skb)) {
                        /* GSO-marked skbs but without frags, handle
                         * them normally
                         */
                        if (skb_shinfo(chunk->skb)->frag_list)
                                chunk->head_skb = chunk->skb;

                        /* skbs with "cover letter" */
                        if (chunk->head_skb && chunk->skb->data_len == chunk->skb->len)
                                chunk->skb = skb_shinfo(chunk->skb)->frag_list;

                        if (WARN_ON(!chunk->skb)) {
                                __SCTP_INC_STATS(dev_net(chunk->skb->dev), SCTP_MIB_IN_PKT_DISCARDS);
                                sctp_chunk_free(chunk);
                                goto next_chunk;
                        }
                }

                if (chunk->asoc)
                        sock_rps_save_rxhash(chunk->asoc->base.sk, chunk->skb);

                queue->in_progress = chunk;

new_skb:
                /* This is the first chunk in the packet.  */
                ch = (struct sctp_chunkhdr *)chunk->skb->data;
                chunk->singleton = 1;
                chunk->data_accepted = 0;
                chunk->pdiscard = 0;
                chunk->auth = 0;
                chunk->has_asconf = 0;
                chunk->end_of_packet = 0;
                if (chunk->head_skb) {
                        struct sctp_input_cb
                                *cb = SCTP_INPUT_CB(chunk->skb),
                                *head_cb = SCTP_INPUT_CB(chunk->head_skb);

                        cb->chunk = head_cb->chunk;
                        cb->af = head_cb->af;
                }
        }

        chunk->chunk_hdr = ch;
        chunk->chunk_end = ((__u8 *)ch) + SCTP_PAD4(ntohs(ch->length));
        skb_pull(chunk->skb, sizeof(*ch));
        chunk->subh.v = NULL; /* Subheader is no longer valid.  */

        if (chunk->chunk_end + sizeof(*ch) <= skb_tail_pointer(chunk->skb)) {
                /* This is not a singleton */
                chunk->singleton = 0;
        } else if (chunk->chunk_end > skb_tail_pointer(chunk->skb)) {
                /* Discard inside state machine. */
                chunk->pdiscard = 1;
                chunk->chunk_end = skb_tail_pointer(chunk->skb);
        } else {
                /* We are at the end of the packet, so mark the chunk
                 * in case we need to send a SACK.
                 */
                chunk->end_of_packet = 1;
        }

        pr_debug("+++sctp_inq_pop+++ chunk:%p[%s], length:%d, skb->len:%d\n",
                 chunk, sctp_cname(SCTP_ST_CHUNK(chunk->chunk_hdr->type)),
                 ntohs(chunk->chunk_hdr->length), chunk->skb->len);

        return chunk;
}

/* Set a top-half handler.
 *
 * Originally, we the top-half handler was scheduled as a BH.  We now
 * call the handler directly in sctp_inq_push() at a time that
 * we know we are lock safe.
 * The intent is that this routine will pull stuff out of the
 * inqueue and process it.
 */
void sctp_inq_set_th_handler(struct sctp_inq *q, work_func_t callback)
{
        INIT_WORK(&q->immediate, callback);
}


















 8594 







 1311 





















  360 




  364 


  364 
  361 


  353 
    1 

  354 
  353 


  350 





  347 
  320 


  351 


    9 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
// SPDX-License-Identifier: GPL-2.0
#include <linux/compiler.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/fault-inject-usercopy.h>
#include <linux/instrumented.h>
#include <linux/kernel.h>
#include <linux/nospec.h>
#include <linux/string.h>
#include <linux/uaccess.h>
#include <linux/wordpart.h>

/* out-of-line parts */

#if !defined(INLINE_COPY_FROM_USER) || defined(CONFIG_RUST)
unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n)
{
        return _inline_copy_from_user(to, from, n);
}
EXPORT_SYMBOL(_copy_from_user);
#endif

#if !defined(INLINE_COPY_TO_USER) || defined(CONFIG_RUST)
unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n)
{
        return _inline_copy_to_user(to, from, n);
}
EXPORT_SYMBOL(_copy_to_user);
#endif

/**
 * check_zeroed_user: check if a userspace buffer only contains zero bytes
 * @from: Source address, in userspace.
 * @size: Size of buffer.
 *
 * This is effectively shorthand for "memchr_inv(from, 0, size) == NULL" for
 * userspace addresses (and is more efficient because we don't care where the
 * first non-zero byte is).
 *
 * Returns:
 *  * 0: There were non-zero bytes present in the buffer.
 *  * 1: The buffer was full of zero bytes.
 *  * -EFAULT: access to userspace failed.
 */
int check_zeroed_user(const void __user *from, size_t size)
{
        unsigned long val;
        uintptr_t align = (uintptr_t) from % sizeof(unsigned long);

        if (unlikely(size == 0))
                return 1;

        from -= align;
        size += align;

        if (!user_read_access_begin(from, size))
                return -EFAULT;

        unsafe_get_user(val, (unsigned long __user *) from, err_fault);
        if (align)
                val &= ~aligned_byte_mask(align);

        while (size > sizeof(unsigned long)) {
                if (unlikely(val))
                        goto done;

                from += sizeof(unsigned long);
                size -= sizeof(unsigned long);

                unsafe_get_user(val, (unsigned long __user *) from, err_fault);
        }

        if (size < sizeof(unsigned long))
                val &= aligned_byte_mask(size);

done:
        user_read_access_end();
        return (val == 0);
err_fault:
        user_read_access_end();
        return -EFAULT;
}
EXPORT_SYMBOL(check_zeroed_user);










































  165 





























  249 














































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 1994 Linus Torvalds
 *
 * Pentium III FXSR, SSE support
 * General FPU state handling cleanups
 *        Gareth Hughes <gareth@valinux.com>, May 2000
 * x86-64 work by Andi Kleen 2002
 */

#ifndef _ASM_X86_FPU_API_H
#define _ASM_X86_FPU_API_H
#include <linux/bottom_half.h>

#include <asm/fpu/types.h>

/*
 * Use kernel_fpu_begin/end() if you intend to use FPU in kernel context. It
 * disables preemption so be careful if you intend to use it for long periods
 * of time.
 * If you intend to use the FPU in irq/softirq you need to check first with
 * irq_fpu_usable() if it is possible.
 */

/* Kernel FPU states to initialize in kernel_fpu_begin_mask() */
#define KFPU_387        _BITUL(0)        /* 387 state will be initialized */
#define KFPU_MXCSR        _BITUL(1)        /* MXCSR will be initialized */

extern void kernel_fpu_begin_mask(unsigned int kfpu_mask);
extern void kernel_fpu_end(void);
extern bool irq_fpu_usable(void);
extern void fpregs_mark_activate(void);

/* Code that is unaware of kernel_fpu_begin_mask() can use this */
static inline void kernel_fpu_begin(void)
{
#ifdef CONFIG_X86_64
        /*
         * Any 64-bit code that uses 387 instructions must explicitly request
         * KFPU_387.
         */
        kernel_fpu_begin_mask(KFPU_MXCSR);
#else
        /*
         * 32-bit kernel code may use 387 operations as well as SSE2, etc,
         * as long as it checks that the CPU has the required capability.
         */
        kernel_fpu_begin_mask(KFPU_387 | KFPU_MXCSR);
#endif
}

/*
 * Use fpregs_lock() while editing CPU's FPU registers or fpu->fpstate.
 * A context switch will (and softirq might) save CPU's FPU registers to
 * fpu->fpstate.regs and set TIF_NEED_FPU_LOAD leaving CPU's FPU registers in
 * a random state.
 *
 * local_bh_disable() protects against both preemption and soft interrupts
 * on !RT kernels.
 *
 * On RT kernels local_bh_disable() is not sufficient because it only
 * serializes soft interrupt related sections via a local lock, but stays
 * preemptible. Disabling preemption is the right choice here as bottom
 * half processing is always in thread context on RT kernels so it
 * implicitly prevents bottom half processing as well.
 *
 * Disabling preemption also serializes against kernel_fpu_begin().
 */
static inline void fpregs_lock(void)
{
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                local_bh_disable();
        else
                preempt_disable();
}

static inline void fpregs_unlock(void)
{
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                local_bh_enable();
        else
                preempt_enable();
}

/*
 * FPU state gets lazily restored before returning to userspace. So when in the
 * kernel, the valid FPU state may be kept in the buffer. This function will force
 * restore all the fpu state to the registers early if needed, and lock them from
 * being automatically saved/restored. Then FPU state can be modified safely in the
 * registers, before unlocking with fpregs_unlock().
 */
void fpregs_lock_and_load(void);

#ifdef CONFIG_X86_DEBUG_FPU
extern void fpregs_assert_state_consistent(void);
#else
static inline void fpregs_assert_state_consistent(void) { }
#endif

/*
 * Load the task FPU state before returning to userspace.
 */
extern void switch_fpu_return(void);

/*
 * Query the presence of one or more xfeatures. Works on any legacy CPU as well.
 *
 * If 'feature_name' is set then put a human-readable description of
 * the feature there as well - this can be used to print error (or success)
 * messages.
 */
extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);

/* Trap handling */
extern int  fpu__exception_code(struct fpu *fpu, int trap_nr);
extern void fpu_sync_fpstate(struct fpu *fpu);
extern void fpu_reset_from_exception_fixup(void);

/* Boot, hotplug and resume */
extern void fpu__init_cpu(void);
extern void fpu__init_system(void);
extern void fpu__init_check_bugs(void);
extern void fpu__resume_cpu(void);

#ifdef CONFIG_MATH_EMULATION
extern void fpstate_init_soft(struct swregs_state *soft);
#else
static inline void fpstate_init_soft(struct swregs_state *soft) {}
#endif

/* State tracking */
DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);

/* Process cleanup */
#ifdef CONFIG_X86_64
extern void fpstate_free(struct fpu *fpu);
#else
static inline void fpstate_free(struct fpu *fpu) { }
#endif

/* fpstate-related functions which are exported to KVM */
extern void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature);

extern u64 xstate_get_guest_group_perm(void);

extern void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr);


/* KVM specific functions */
extern bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu);
extern void fpu_free_guest_fpstate(struct fpu_guest *gfpu);
extern int fpu_swap_kvm_fpstate(struct fpu_guest *gfpu, bool enter_guest);
extern int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures);

#ifdef CONFIG_X86_64
extern void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd);
extern void fpu_sync_guest_vmexit_xfd_state(void);
#else
static inline void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd) { }
static inline void fpu_sync_guest_vmexit_xfd_state(void) { }
#endif

extern void fpu_copy_guest_fpstate_to_uabi(struct fpu_guest *gfpu, void *buf,
                                           unsigned int size, u64 xfeatures, u32 pkru);
extern int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, u64 xcr0, u32 *vpkru);

static inline void fpstate_set_confidential(struct fpu_guest *gfpu)
{
        gfpu->fpstate->is_confidential = true;
}

static inline bool fpstate_is_confidential(struct fpu_guest *gfpu)
{
        return gfpu->fpstate->is_confidential;
}

/* prctl */
extern long fpu_xstate_prctl(int option, unsigned long arg2);

extern void fpu_idle_fpregs(void);

#endif /* _ASM_X86_FPU_API_H */





















































































































  326 







  324 



  326 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Kernel Electric-Fence (KFENCE). For more info please see
 * Documentation/dev-tools/kfence.rst.
 *
 * Copyright (C) 2020, Google LLC.
 */

#ifndef MM_KFENCE_KFENCE_H
#define MM_KFENCE_KFENCE_H

#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>

#include "../slab.h" /* for struct kmem_cache */

/*
 * Get the canary byte pattern for @addr. Use a pattern that varies based on the
 * lower 3 bits of the address, to detect memory corruptions with higher
 * probability, where similar constants are used.
 */
#define KFENCE_CANARY_PATTERN_U8(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7))

/*
 * Define a continuous 8-byte canary starting from a multiple of 8. The canary
 * of each byte is only related to the lowest three bits of its address, so the
 * canary of every 8 bytes is the same. 64-bit memory can be filled and checked
 * at a time instead of byte by byte to improve performance.
 */
#define KFENCE_CANARY_PATTERN_U64 ((u64)0xaaaaaaaaaaaaaaaa ^ (u64)(le64_to_cpu(0x0706050403020100)))

/* Maximum stack depth for reports. */
#define KFENCE_STACK_DEPTH 64

/* KFENCE object states. */
enum kfence_object_state {
        KFENCE_OBJECT_UNUSED,                /* Object is unused. */
        KFENCE_OBJECT_ALLOCATED,        /* Object is currently allocated. */
        KFENCE_OBJECT_RCU_FREEING,        /* Object was allocated, and then being freed by rcu. */
        KFENCE_OBJECT_FREED,                /* Object was allocated, and then freed. */
};

/* Alloc/free tracking information. */
struct kfence_track {
        pid_t pid;
        int cpu;
        u64 ts_nsec;
        int num_stack_entries;
        unsigned long stack_entries[KFENCE_STACK_DEPTH];
};

/* KFENCE metadata per guarded allocation. */
struct kfence_metadata {
        struct list_head list;                /* Freelist node; access under kfence_freelist_lock. */
        struct rcu_head rcu_head;        /* For delayed freeing. */

        /*
         * Lock protecting below data; to ensure consistency of the below data,
         * since the following may execute concurrently: __kfence_alloc(),
         * __kfence_free(), kfence_handle_page_fault(). However, note that we
         * cannot grab the same metadata off the freelist twice, and multiple
         * __kfence_alloc() cannot run concurrently on the same metadata.
         */
        raw_spinlock_t lock;

        /* The current state of the object; see above. */
        enum kfence_object_state state;

        /*
         * Allocated object address; cannot be calculated from size, because of
         * alignment requirements.
         *
         * Invariant: ALIGN_DOWN(addr, PAGE_SIZE) is constant.
         */
        unsigned long addr;

        /*
         * The size of the original allocation.
         */
        size_t size;

        /*
         * The kmem_cache cache of the last allocation; NULL if never allocated
         * or the cache has already been destroyed.
         */
        struct kmem_cache *cache;

        /*
         * In case of an invalid access, the page that was unprotected; we
         * optimistically only store one address.
         */
        unsigned long unprotected_page;

        /* Allocation and free stack information. */
        struct kfence_track alloc_track;
        struct kfence_track free_track;
        /* For updating alloc_covered on frees. */
        u32 alloc_stack_hash;
#ifdef CONFIG_MEMCG
        struct slabobj_ext obj_exts;
#endif
};

#define KFENCE_METADATA_SIZE PAGE_ALIGN(sizeof(struct kfence_metadata) * \
                                        CONFIG_KFENCE_NUM_OBJECTS)

extern struct kfence_metadata *kfence_metadata;

static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)
{
        long index;

        /* The checks do not affect performance; only called from slow-paths. */

        if (!is_kfence_address((void *)addr))
                return NULL;

        /*
         * May be an invalid index if called with an address at the edge of
         * __kfence_pool, in which case we would report an "invalid access"
         * error.
         */
        index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;
        if (index < 0 || index >= CONFIG_KFENCE_NUM_OBJECTS)
                return NULL;

        return &kfence_metadata[index];
}

/* KFENCE error types for report generation. */
enum kfence_error_type {
        KFENCE_ERROR_OOB,                /* Detected a out-of-bounds access. */
        KFENCE_ERROR_UAF,                /* Detected a use-after-free access. */
        KFENCE_ERROR_CORRUPTION,        /* Detected a memory corruption on free. */
        KFENCE_ERROR_INVALID,                /* Invalid access of unknown type. */
        KFENCE_ERROR_INVALID_FREE,        /* Invalid free. */
};

void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
                         const struct kfence_metadata *meta, enum kfence_error_type type);

void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta);

#endif /* MM_KFENCE_KFENCE_H */




















































































































































  324 









  124 







































































































































   85 





   16 










































   27 


















   27 






























    3 

































   12 

















   18 










  458 







   16 

    9 
   39 
























   10 

  134 
   44 
















  134 















    8 
























  308 









  306 




























   26 








   26 









   47 





























  223 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_GENERIC_NETLINK_H
#define __NET_GENERIC_NETLINK_H

#include <linux/net.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#include <uapi/linux/genetlink.h>

#define GENLMSG_DEFAULT_SIZE (NLMSG_DEFAULT_SIZE - GENL_HDRLEN)

/* Non-parallel generic netlink requests are serialized by a global lock. */
void genl_lock(void);
void genl_unlock(void);

#define MODULE_ALIAS_GENL_FAMILY(family) \
 MODULE_ALIAS_NET_PF_PROTO_NAME(PF_NETLINK, NETLINK_GENERIC, "-family-" family)

/* Binding to multicast group requires %CAP_NET_ADMIN */
#define GENL_MCAST_CAP_NET_ADMIN        BIT(0)
/* Binding to multicast group requires %CAP_SYS_ADMIN */
#define GENL_MCAST_CAP_SYS_ADMIN        BIT(1)

/**
 * struct genl_multicast_group - generic netlink multicast group
 * @name: name of the multicast group, names are per-family
 * @flags: GENL_MCAST_* flags
 */
struct genl_multicast_group {
        char                        name[GENL_NAMSIZ];
        u8                        flags;
};

struct genl_split_ops;
struct genl_info;

/**
 * struct genl_family - generic netlink family
 * @hdrsize: length of user specific header in bytes
 * @name: name of family
 * @version: protocol version
 * @maxattr: maximum number of attributes supported
 * @policy: netlink policy
 * @netnsok: set to true if the family can handle network
 *        namespaces and should be presented in all of them
 * @parallel_ops: operations can be called in parallel and aren't
 *        synchronized by the core genetlink code
 * @pre_doit: called before an operation's doit callback, it may
 *        do additional, common, filtering and return an error
 * @post_doit: called after an operation's doit callback, it may
 *        undo operations done by pre_doit, for example release locks
 * @bind: called when family multicast group is added to a netlink socket
 * @unbind: called when family multicast group is removed from a netlink socket
 * @module: pointer to the owning module (set to THIS_MODULE)
 * @mcgrps: multicast groups used by this family
 * @n_mcgrps: number of multicast groups
 * @resv_start_op: first operation for which reserved fields of the header
 *        can be validated and policies are required (see below);
 *        new families should leave this field at zero
 * @ops: the operations supported by this family
 * @n_ops: number of operations supported by this family
 * @small_ops: the small-struct operations supported by this family
 * @n_small_ops: number of small-struct operations supported by this family
 * @split_ops: the split do/dump form of operation definition
 * @n_split_ops: number of entries in @split_ops, not that with split do/dump
 *        ops the number of entries is not the same as number of commands
 * @sock_priv_size: the size of per-socket private memory
 * @sock_priv_init: the per-socket private memory initializer
 * @sock_priv_destroy: the per-socket private memory destructor
 *
 * Attribute policies (the combination of @policy and @maxattr fields)
 * can be attached at the family level or at the operation level.
 * If both are present the per-operation policy takes precedence.
 * For operations before @resv_start_op lack of policy means that the core
 * will perform no attribute parsing or validation. For newer operations
 * if policy is not provided core will reject all TLV attributes.
 */
struct genl_family {
        unsigned int                hdrsize;
        char                        name[GENL_NAMSIZ];
        unsigned int                version;
        unsigned int                maxattr;
        u8                        netnsok:1;
        u8                        parallel_ops:1;
        u8                        n_ops;
        u8                        n_small_ops;
        u8                        n_split_ops;
        u8                        n_mcgrps;
        u8                        resv_start_op;
        const struct nla_policy *policy;
        int                        (*pre_doit)(const struct genl_split_ops *ops,
                                            struct sk_buff *skb,
                                            struct genl_info *info);
        void                        (*post_doit)(const struct genl_split_ops *ops,
                                             struct sk_buff *skb,
                                             struct genl_info *info);
        int                        (*bind)(int mcgrp);
        void                        (*unbind)(int mcgrp);
        const struct genl_ops *        ops;
        const struct genl_small_ops *small_ops;
        const struct genl_split_ops *split_ops;
        const struct genl_multicast_group *mcgrps;
        struct module                *module;

        size_t                        sock_priv_size;
        void                        (*sock_priv_init)(void *priv);
        void                        (*sock_priv_destroy)(void *priv);

/* private: internal use only */
        /* protocol family identifier */
        int                        id;
        /* starting number of multicast group IDs in this family */
        unsigned int                mcgrp_offset;
        /* list of per-socket privs */
        struct xarray                *sock_privs;
};

/**
 * struct genl_info - receiving information
 * @snd_seq: sending sequence number
 * @snd_portid: netlink portid of sender
 * @family: generic netlink family
 * @nlhdr: netlink message header
 * @genlhdr: generic netlink message header
 * @attrs: netlink attributes
 * @_net: network namespace
 * @ctx: storage space for the use by the family
 * @user_ptr: user pointers (deprecated, use ctx instead)
 * @extack: extended ACK report struct
 */
struct genl_info {
        u32                        snd_seq;
        u32                        snd_portid;
        const struct genl_family *family;
        const struct nlmsghdr *        nlhdr;
        struct genlmsghdr *        genlhdr;
        struct nlattr **        attrs;
        possible_net_t                _net;
        union {
                u8                ctx[NETLINK_CTX_SIZE];
                void *                user_ptr[2];
        };
        struct netlink_ext_ack *extack;
};

static inline struct net *genl_info_net(const struct genl_info *info)
{
        return read_pnet(&info->_net);
}

static inline void genl_info_net_set(struct genl_info *info, struct net *net)
{
        write_pnet(&info->_net, net);
}

static inline void *genl_info_userhdr(const struct genl_info *info)
{
        return (u8 *)info->genlhdr + GENL_HDRLEN;
}

#define GENL_SET_ERR_MSG(info, msg) NL_SET_ERR_MSG((info)->extack, msg)

#define GENL_SET_ERR_MSG_FMT(info, msg, args...) \
        NL_SET_ERR_MSG_FMT((info)->extack, msg, ##args)

/* Report that a root attribute is missing */
#define GENL_REQ_ATTR_CHECK(info, attr) ({                                \
        const struct genl_info *__info = (info);                        \
                                                                        \
        NL_REQ_ATTR_CHECK(__info->extack, NULL, __info->attrs, (attr)); \
})

enum genl_validate_flags {
        GENL_DONT_VALIDATE_STRICT                = BIT(0),
        GENL_DONT_VALIDATE_DUMP                        = BIT(1),
        GENL_DONT_VALIDATE_DUMP_STRICT                = BIT(2),
};

/**
 * struct genl_small_ops - generic netlink operations (small version)
 * @cmd: command identifier
 * @internal_flags: flags used by the family
 * @flags: GENL_* flags (%GENL_ADMIN_PERM or %GENL_UNS_ADMIN_PERM)
 * @validate: validation flags from enum genl_validate_flags
 * @doit: standard command callback
 * @dumpit: callback for dumpers
 *
 * This is a cut-down version of struct genl_ops for users who don't need
 * most of the ancillary infra and want to save space.
 */
struct genl_small_ops {
        int        (*doit)(struct sk_buff *skb, struct genl_info *info);
        int        (*dumpit)(struct sk_buff *skb, struct netlink_callback *cb);
        u8        cmd;
        u8        internal_flags;
        u8        flags;
        u8        validate;
};

/**
 * struct genl_ops - generic netlink operations
 * @cmd: command identifier
 * @internal_flags: flags used by the family
 * @flags: GENL_* flags (%GENL_ADMIN_PERM or %GENL_UNS_ADMIN_PERM)
 * @maxattr: maximum number of attributes supported
 * @policy: netlink policy (takes precedence over family policy)
 * @validate: validation flags from enum genl_validate_flags
 * @doit: standard command callback
 * @start: start callback for dumps
 * @dumpit: callback for dumpers
 * @done: completion callback for dumps
 */
struct genl_ops {
        int                       (*doit)(struct sk_buff *skb,
                                       struct genl_info *info);
        int                       (*start)(struct netlink_callback *cb);
        int                       (*dumpit)(struct sk_buff *skb,
                                         struct netlink_callback *cb);
        int                       (*done)(struct netlink_callback *cb);
        const struct nla_policy *policy;
        unsigned int                maxattr;
        u8                        cmd;
        u8                        internal_flags;
        u8                        flags;
        u8                        validate;
};

/**
 * struct genl_split_ops - generic netlink operations (do/dump split version)
 * @cmd: command identifier
 * @internal_flags: flags used by the family
 * @flags: GENL_* flags (%GENL_ADMIN_PERM or %GENL_UNS_ADMIN_PERM)
 * @validate: validation flags from enum genl_validate_flags
 * @policy: netlink policy (takes precedence over family policy)
 * @maxattr: maximum number of attributes supported
 *
 * Do callbacks:
 * @pre_doit: called before an operation's @doit callback, it may
 *        do additional, common, filtering and return an error
 * @doit: standard command callback
 * @post_doit: called after an operation's @doit callback, it may
 *        undo operations done by pre_doit, for example release locks
 *
 * Dump callbacks:
 * @start: start callback for dumps
 * @dumpit: callback for dumpers
 * @done: completion callback for dumps
 *
 * Do callbacks can be used if %GENL_CMD_CAP_DO is set in @flags.
 * Dump callbacks can be used if %GENL_CMD_CAP_DUMP is set in @flags.
 * Exactly one of those flags must be set.
 */
struct genl_split_ops {
        union {
                struct {
                        int (*pre_doit)(const struct genl_split_ops *ops,
                                        struct sk_buff *skb,
                                        struct genl_info *info);
                        int (*doit)(struct sk_buff *skb,
                                    struct genl_info *info);
                        void (*post_doit)(const struct genl_split_ops *ops,
                                          struct sk_buff *skb,
                                          struct genl_info *info);
                };
                struct {
                        int (*start)(struct netlink_callback *cb);
                        int (*dumpit)(struct sk_buff *skb,
                                      struct netlink_callback *cb);
                        int (*done)(struct netlink_callback *cb);
                };
        };
        const struct nla_policy *policy;
        unsigned int                maxattr;
        u8                        cmd;
        u8                        internal_flags;
        u8                        flags;
        u8                        validate;
};

/**
 * struct genl_dumpit_info - info that is available during dumpit op call
 * @op: generic netlink ops - for internal genl code usage
 * @attrs: netlink attributes
 * @info: struct genl_info describing the request
 */
struct genl_dumpit_info {
        struct genl_split_ops op;
        struct genl_info info;
};

static inline const struct genl_dumpit_info *
genl_dumpit_info(struct netlink_callback *cb)
{
        return cb->data;
}

static inline const struct genl_info *
genl_info_dump(struct netlink_callback *cb)
{
        return &genl_dumpit_info(cb)->info;
}

/**
 * genl_info_init_ntf() - initialize genl_info for notifications
 * @info:   genl_info struct to set up
 * @family: pointer to the genetlink family
 * @cmd:    command to be used in the notification
 *
 * Initialize a locally declared struct genl_info to pass to various APIs.
 * Intended to be used when creating notifications.
 */
static inline void
genl_info_init_ntf(struct genl_info *info, const struct genl_family *family,
                   u8 cmd)
{
        struct genlmsghdr *hdr = (void *) &info->user_ptr[0];

        memset(info, 0, sizeof(*info));
        info->family = family;
        info->genlhdr = hdr;
        hdr->cmd = cmd;
}

static inline bool genl_info_is_ntf(const struct genl_info *info)
{
        return !info->nlhdr;
}

void *__genl_sk_priv_get(struct genl_family *family, struct sock *sk);
void *genl_sk_priv_get(struct genl_family *family, struct sock *sk);
int genl_register_family(struct genl_family *family);
int genl_unregister_family(const struct genl_family *family);
void genl_notify(const struct genl_family *family, struct sk_buff *skb,
                 struct genl_info *info, u32 group, gfp_t flags);

void *genlmsg_put(struct sk_buff *skb, u32 portid, u32 seq,
                  const struct genl_family *family, int flags, u8 cmd);

static inline void *
__genlmsg_iput(struct sk_buff *skb, const struct genl_info *info, int flags)
{
        return genlmsg_put(skb, info->snd_portid, info->snd_seq, info->family,
                           flags, info->genlhdr->cmd);
}

/**
 * genlmsg_iput - start genetlink message based on genl_info
 * @skb: skb in which message header will be placed
 * @info: genl_info as provided to do/dump handlers
 *
 * Convenience wrapper which starts a genetlink message based on
 * information in user request. @info should be either the struct passed
 * by genetlink core to do/dump handlers (when constructing replies to
 * such requests) or a struct initialized by genl_info_init_ntf()
 * when constructing notifications.
 *
 * Returns pointer to new genetlink header.
 */
static inline void *
genlmsg_iput(struct sk_buff *skb, const struct genl_info *info)
{
        return __genlmsg_iput(skb, info, 0);
}

/**
 * genlmsg_nlhdr - Obtain netlink header from user specified header
 * @user_hdr: user header as returned from genlmsg_put()
 *
 * Returns pointer to netlink header.
 */
static inline struct nlmsghdr *genlmsg_nlhdr(void *user_hdr)
{
        return (struct nlmsghdr *)((char *)user_hdr -
                                   GENL_HDRLEN -
                                   NLMSG_HDRLEN);
}

/**
 * genlmsg_parse_deprecated - parse attributes of a genetlink message
 * @nlh: netlink message header
 * @family: genetlink message family
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @policy: validation policy
 * @extack: extended ACK report struct
 */
static inline int genlmsg_parse_deprecated(const struct nlmsghdr *nlh,
                                           const struct genl_family *family,
                                           struct nlattr *tb[], int maxtype,
                                           const struct nla_policy *policy,
                                           struct netlink_ext_ack *extack)
{
        return __nlmsg_parse(nlh, family->hdrsize + GENL_HDRLEN, tb, maxtype,
                             policy, NL_VALIDATE_LIBERAL, extack);
}

/**
 * genlmsg_parse - parse attributes of a genetlink message
 * @nlh: netlink message header
 * @family: genetlink message family
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @policy: validation policy
 * @extack: extended ACK report struct
 */
static inline int genlmsg_parse(const struct nlmsghdr *nlh,
                                const struct genl_family *family,
                                struct nlattr *tb[], int maxtype,
                                const struct nla_policy *policy,
                                struct netlink_ext_ack *extack)
{
        return __nlmsg_parse(nlh, family->hdrsize + GENL_HDRLEN, tb, maxtype,
                             policy, NL_VALIDATE_STRICT, extack);
}

/**
 * genl_dump_check_consistent - check if sequence is consistent and advertise if not
 * @cb: netlink callback structure that stores the sequence number
 * @user_hdr: user header as returned from genlmsg_put()
 *
 * Cf. nl_dump_check_consistent(), this just provides a wrapper to make it
 * simpler to use with generic netlink.
 */
static inline void genl_dump_check_consistent(struct netlink_callback *cb,
                                              void *user_hdr)
{
        nl_dump_check_consistent(cb, genlmsg_nlhdr(user_hdr));
}

/**
 * genlmsg_put_reply - Add generic netlink header to a reply message
 * @skb: socket buffer holding the message
 * @info: receiver info
 * @family: generic netlink family
 * @flags: netlink message flags
 * @cmd: generic netlink command
 *
 * Returns pointer to user specific header
 */
static inline void *genlmsg_put_reply(struct sk_buff *skb,
                                      struct genl_info *info,
                                      const struct genl_family *family,
                                      int flags, u8 cmd)
{
        return genlmsg_put(skb, info->snd_portid, info->snd_seq, family,
                           flags, cmd);
}

/**
 * genlmsg_end - Finalize a generic netlink message
 * @skb: socket buffer the message is stored in
 * @hdr: user specific header
 */
static inline void genlmsg_end(struct sk_buff *skb, void *hdr)
{
        nlmsg_end(skb, hdr - GENL_HDRLEN - NLMSG_HDRLEN);
}

/**
 * genlmsg_cancel - Cancel construction of a generic netlink message
 * @skb: socket buffer the message is stored in
 * @hdr: generic netlink message header
 */
static inline void genlmsg_cancel(struct sk_buff *skb, void *hdr)
{
        if (hdr)
                nlmsg_cancel(skb, hdr - GENL_HDRLEN - NLMSG_HDRLEN);
}

/**
 * genlmsg_multicast_netns_filtered - multicast a netlink message
 *                                      to a specific netns with filter
 *                                      function
 * @family: the generic netlink family
 * @net: the net namespace
 * @skb: netlink message as socket buffer
 * @portid: own netlink portid to avoid sending to yourself
 * @group: offset of multicast group in groups array
 * @flags: allocation flags
 * @filter: filter function
 * @filter_data: filter function private data
 *
 * Return: 0 on success, negative error code for failure.
 */
static inline int
genlmsg_multicast_netns_filtered(const struct genl_family *family,
                                 struct net *net, struct sk_buff *skb,
                                 u32 portid, unsigned int group, gfp_t flags,
                                 netlink_filter_fn filter,
                                 void *filter_data)
{
        if (WARN_ON_ONCE(group >= family->n_mcgrps))
                return -EINVAL;
        group = family->mcgrp_offset + group;
        return nlmsg_multicast_filtered(net->genl_sock, skb, portid, group,
                                        flags, filter, filter_data);
}

/**
 * genlmsg_multicast_netns - multicast a netlink message to a specific netns
 * @family: the generic netlink family
 * @net: the net namespace
 * @skb: netlink message as socket buffer
 * @portid: own netlink portid to avoid sending to yourself
 * @group: offset of multicast group in groups array
 * @flags: allocation flags
 */
static inline int genlmsg_multicast_netns(const struct genl_family *family,
                                          struct net *net, struct sk_buff *skb,
                                          u32 portid, unsigned int group, gfp_t flags)
{
        return genlmsg_multicast_netns_filtered(family, net, skb, portid,
                                                group, flags, NULL, NULL);
}

/**
 * genlmsg_multicast - multicast a netlink message to the default netns
 * @family: the generic netlink family
 * @skb: netlink message as socket buffer
 * @portid: own netlink portid to avoid sending to yourself
 * @group: offset of multicast group in groups array
 * @flags: allocation flags
 */
static inline int genlmsg_multicast(const struct genl_family *family,
                                    struct sk_buff *skb, u32 portid,
                                    unsigned int group, gfp_t flags)
{
        return genlmsg_multicast_netns(family, &init_net, skb,
                                       portid, group, flags);
}

/**
 * genlmsg_multicast_allns - multicast a netlink message to all net namespaces
 * @family: the generic netlink family
 * @skb: netlink message as socket buffer
 * @portid: own netlink portid to avoid sending to yourself
 * @group: offset of multicast group in groups array
 *
 * This function must hold the RTNL or rcu_read_lock().
 */
int genlmsg_multicast_allns(const struct genl_family *family,
                            struct sk_buff *skb, u32 portid,
                            unsigned int group);

/**
 * genlmsg_unicast - unicast a netlink message
 * @net: network namespace to look up @portid in
 * @skb: netlink message as socket buffer
 * @portid: netlink portid of the destination socket
 */
static inline int genlmsg_unicast(struct net *net, struct sk_buff *skb, u32 portid)
{
        return nlmsg_unicast(net->genl_sock, skb, portid);
}

/**
 * genlmsg_reply - reply to a request
 * @skb: netlink message to be sent back
 * @info: receiver information
 */
static inline int genlmsg_reply(struct sk_buff *skb, struct genl_info *info)
{
        return genlmsg_unicast(genl_info_net(info), skb, info->snd_portid);
}

/**
 * genlmsg_data - head of message payload
 * @gnlh: genetlink message header
 */
static inline void *genlmsg_data(const struct genlmsghdr *gnlh)
{
        return ((unsigned char *) gnlh + GENL_HDRLEN);
}

/**
 * genlmsg_len - length of message payload
 * @gnlh: genetlink message header
 */
static inline int genlmsg_len(const struct genlmsghdr *gnlh)
{
        struct nlmsghdr *nlh = (struct nlmsghdr *)((unsigned char *)gnlh -
                                                        NLMSG_HDRLEN);
        return (nlh->nlmsg_len - GENL_HDRLEN - NLMSG_HDRLEN);
}

/**
 * genlmsg_msg_size - length of genetlink message not including padding
 * @payload: length of message payload
 */
static inline int genlmsg_msg_size(int payload)
{
        return GENL_HDRLEN + payload;
}

/**
 * genlmsg_total_size - length of genetlink message including padding
 * @payload: length of message payload
 */
static inline int genlmsg_total_size(int payload)
{
        return NLMSG_ALIGN(genlmsg_msg_size(payload));
}

/**
 * genlmsg_new - Allocate a new generic netlink message
 * @payload: size of the message payload
 * @flags: the type of memory to allocate.
 */
static inline struct sk_buff *genlmsg_new(size_t payload, gfp_t flags)
{
        return nlmsg_new(genlmsg_total_size(payload), flags);
}

/**
 * genl_set_err - report error to genetlink broadcast listeners
 * @family: the generic netlink family
 * @net: the network namespace to report the error to
 * @portid: the PORTID of a process that we want to skip (if any)
 * @group: the broadcast group that will notice the error
 *         (this is the offset of the multicast group in the groups array)
 * @code: error code, must be negative (as usual in kernelspace)
 *
 * This function returns the number of broadcast listeners that have set the
 * NETLINK_RECV_NO_ENOBUFS socket option.
 */
static inline int genl_set_err(const struct genl_family *family,
                               struct net *net, u32 portid,
                               u32 group, int code)
{
        if (WARN_ON_ONCE(group >= family->n_mcgrps))
                return -EINVAL;
        group = family->mcgrp_offset + group;
        return netlink_set_err(net->genl_sock, portid, group, code);
}

static inline int genl_has_listeners(const struct genl_family *family,
                                     struct net *net, unsigned int group)
{
        if (WARN_ON_ONCE(group >= family->n_mcgrps))
                return -EINVAL;
        group = family->mcgrp_offset + group;
        return netlink_has_listeners(net->genl_sock, group);
}
#endif        /* __NET_GENERIC_NETLINK_H */




































































   37 




   35 
   35 
   34 









   35 





   35 
















   35 
   35 






















   14 
   14 




   17 
   17 




   17 


































































    2 
    2 
    2 















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_DCCP_H
#define _LINUX_DCCP_H


#include <linux/in.h>
#include <linux/interrupt.h>
#include <linux/ktime.h>
#include <linux/list.h>
#include <linux/uio.h>
#include <linux/workqueue.h>

#include <net/inet_connection_sock.h>
#include <net/inet_sock.h>
#include <net/inet_timewait_sock.h>
#include <net/tcp_states.h>
#include <uapi/linux/dccp.h>

enum dccp_state {
        DCCP_OPEN             = TCP_ESTABLISHED,
        DCCP_REQUESTING             = TCP_SYN_SENT,
        DCCP_LISTEN             = TCP_LISTEN,
        DCCP_RESPOND             = TCP_SYN_RECV,
        /*
         * States involved in closing a DCCP connection:
         * 1) ACTIVE_CLOSEREQ is entered by a server sending a CloseReq.
         *
         * 2) CLOSING can have three different meanings (RFC 4340, 8.3):
         *  a. Client has performed active-close, has sent a Close to the server
         *     from state OPEN or PARTOPEN, and is waiting for the final Reset
         *     (in this case, SOCK_DONE == 1).
         *  b. Client is asked to perform passive-close, by receiving a CloseReq
         *     in (PART)OPEN state. It sends a Close and waits for final Reset
         *     (in this case, SOCK_DONE == 0).
         *  c. Server performs an active-close as in (a), keeps TIMEWAIT state.
         *
         * 3) The following intermediate states are employed to give passively
         *    closing nodes a chance to process their unread data:
         *    - PASSIVE_CLOSE    (from OPEN => CLOSED) and
         *    - PASSIVE_CLOSEREQ (from (PART)OPEN to CLOSING; case (b) above).
         */
        DCCP_ACTIVE_CLOSEREQ = TCP_FIN_WAIT1,
        DCCP_PASSIVE_CLOSE   = TCP_CLOSE_WAIT,        /* any node receiving a Close */
        DCCP_CLOSING             = TCP_CLOSING,
        DCCP_TIME_WAIT             = TCP_TIME_WAIT,
        DCCP_CLOSED             = TCP_CLOSE,
        DCCP_NEW_SYN_RECV    = TCP_NEW_SYN_RECV,
        DCCP_PARTOPEN             = TCP_MAX_STATES,
        DCCP_PASSIVE_CLOSEREQ,                        /* clients receiving CloseReq */
        DCCP_MAX_STATES
};

enum {
        DCCPF_OPEN              = TCPF_ESTABLISHED,
        DCCPF_REQUESTING      = TCPF_SYN_SENT,
        DCCPF_LISTEN              = TCPF_LISTEN,
        DCCPF_RESPOND              = TCPF_SYN_RECV,
        DCCPF_ACTIVE_CLOSEREQ = TCPF_FIN_WAIT1,
        DCCPF_CLOSING              = TCPF_CLOSING,
        DCCPF_TIME_WAIT              = TCPF_TIME_WAIT,
        DCCPF_CLOSED              = TCPF_CLOSE,
        DCCPF_NEW_SYN_RECV    = TCPF_NEW_SYN_RECV,
        DCCPF_PARTOPEN              = (1 << DCCP_PARTOPEN),
};

static inline struct dccp_hdr *dccp_hdr(const struct sk_buff *skb)
{
        return (struct dccp_hdr *)skb_transport_header(skb);
}

static inline struct dccp_hdr *dccp_zeroed_hdr(struct sk_buff *skb, int headlen)
{
        skb_push(skb, headlen);
        skb_reset_transport_header(skb);
        return memset(skb_transport_header(skb), 0, headlen);
}

static inline struct dccp_hdr_ext *dccp_hdrx(const struct dccp_hdr *dh)
{
        return (struct dccp_hdr_ext *)((unsigned char *)dh + sizeof(*dh));
}

static inline unsigned int __dccp_basic_hdr_len(const struct dccp_hdr *dh)
{
        return sizeof(*dh) + (dh->dccph_x ? sizeof(struct dccp_hdr_ext) : 0);
}

static inline unsigned int dccp_basic_hdr_len(const struct sk_buff *skb)
{
        const struct dccp_hdr *dh = dccp_hdr(skb);
        return __dccp_basic_hdr_len(dh);
}

static inline __u64 dccp_hdr_seq(const struct dccp_hdr *dh)
{
        __u64 seq_nr =  ntohs(dh->dccph_seq);

        if (dh->dccph_x != 0)
                seq_nr = (seq_nr << 32) + ntohl(dccp_hdrx(dh)->dccph_seq_low);
        else
                seq_nr += (u32)dh->dccph_seq2 << 16;

        return seq_nr;
}

static inline struct dccp_hdr_request *dccp_hdr_request(struct sk_buff *skb)
{
        return (struct dccp_hdr_request *)(skb_transport_header(skb) +
                                           dccp_basic_hdr_len(skb));
}

static inline struct dccp_hdr_ack_bits *dccp_hdr_ack_bits(const struct sk_buff *skb)
{
        return (struct dccp_hdr_ack_bits *)(skb_transport_header(skb) +
                                            dccp_basic_hdr_len(skb));
}

static inline u64 dccp_hdr_ack_seq(const struct sk_buff *skb)
{
        const struct dccp_hdr_ack_bits *dhack = dccp_hdr_ack_bits(skb);
        return ((u64)ntohs(dhack->dccph_ack_nr_high) << 32) + ntohl(dhack->dccph_ack_nr_low);
}

static inline struct dccp_hdr_response *dccp_hdr_response(struct sk_buff *skb)
{
        return (struct dccp_hdr_response *)(skb_transport_header(skb) +
                                            dccp_basic_hdr_len(skb));
}

static inline struct dccp_hdr_reset *dccp_hdr_reset(struct sk_buff *skb)
{
        return (struct dccp_hdr_reset *)(skb_transport_header(skb) +
                                         dccp_basic_hdr_len(skb));
}

static inline unsigned int __dccp_hdr_len(const struct dccp_hdr *dh)
{
        return __dccp_basic_hdr_len(dh) +
               dccp_packet_hdr_len(dh->dccph_type);
}

static inline unsigned int dccp_hdr_len(const struct sk_buff *skb)
{
        return __dccp_hdr_len(dccp_hdr(skb));
}

/**
 * struct dccp_request_sock  -  represent DCCP-specific connection request
 * @dreq_inet_rsk: structure inherited from
 * @dreq_iss: initial sequence number, sent on the first Response (RFC 4340, 7.1)
 * @dreq_gss: greatest sequence number sent (for retransmitted Responses)
 * @dreq_isr: initial sequence number received in the first Request
 * @dreq_gsr: greatest sequence number received (for retransmitted Request(s))
 * @dreq_service: service code present on the Request (there is just one)
 * @dreq_featneg: feature negotiation options for this connection
 * The following two fields are analogous to the ones in dccp_sock:
 * @dreq_timestamp_echo: last received timestamp to echo (13.1)
 * @dreq_timestamp_echo: the time of receiving the last @dreq_timestamp_echo
 */
struct dccp_request_sock {
        struct inet_request_sock dreq_inet_rsk;
        __u64                         dreq_iss;
        __u64                         dreq_gss;
        __u64                         dreq_isr;
        __u64                         dreq_gsr;
        __be32                         dreq_service;
        spinlock_t                 dreq_lock;
        struct list_head         dreq_featneg;
        __u32                         dreq_timestamp_echo;
        __u32                         dreq_timestamp_time;
};

static inline struct dccp_request_sock *dccp_rsk(const struct request_sock *req)
{
        return (struct dccp_request_sock *)req;
}

extern struct inet_timewait_death_row dccp_death_row;

extern int dccp_parse_options(struct sock *sk, struct dccp_request_sock *dreq,
                              struct sk_buff *skb);

struct dccp_options_received {
        u64        dccpor_ndp:48;
        u32        dccpor_timestamp;
        u32        dccpor_timestamp_echo;
        u32        dccpor_elapsed_time;
};

struct ccid;

enum dccp_role {
        DCCP_ROLE_UNDEFINED,
        DCCP_ROLE_LISTEN,
        DCCP_ROLE_CLIENT,
        DCCP_ROLE_SERVER,
};

struct dccp_service_list {
        __u32        dccpsl_nr;
        __be32        dccpsl_list[];
};

#define DCCP_SERVICE_INVALID_VALUE htonl((__u32)-1)
#define DCCP_SERVICE_CODE_IS_ABSENT                0

static inline bool dccp_list_has_service(const struct dccp_service_list *sl,
                                        const __be32 service)
{
        if (likely(sl != NULL)) {
                u32 i = sl->dccpsl_nr;
                while (i--)
                        if (sl->dccpsl_list[i] == service)
                                return true;
        }
        return false;
}

struct dccp_ackvec;

/**
 * struct dccp_sock - DCCP socket state
 *
 * @dccps_swl - sequence number window low
 * @dccps_swh - sequence number window high
 * @dccps_awl - acknowledgement number window low
 * @dccps_awh - acknowledgement number window high
 * @dccps_iss - initial sequence number sent
 * @dccps_isr - initial sequence number received
 * @dccps_osr - first OPEN sequence number received
 * @dccps_gss - greatest sequence number sent
 * @dccps_gsr - greatest valid sequence number received
 * @dccps_gar - greatest valid ack number received on a non-Sync; initialized to %dccps_iss
 * @dccps_service - first (passive sock) or unique (active sock) service code
 * @dccps_service_list - second .. last service code on passive socket
 * @dccps_timestamp_echo - latest timestamp received on a TIMESTAMP option
 * @dccps_timestamp_time - time of receiving latest @dccps_timestamp_echo
 * @dccps_l_ack_ratio - feature-local Ack Ratio
 * @dccps_r_ack_ratio - feature-remote Ack Ratio
 * @dccps_l_seq_win - local Sequence Window (influences ack number validity)
 * @dccps_r_seq_win - remote Sequence Window (influences seq number validity)
 * @dccps_pcslen - sender   partial checksum coverage (via sockopt)
 * @dccps_pcrlen - receiver partial checksum coverage (via sockopt)
 * @dccps_send_ndp_count - local Send NDP Count feature (7.7.2)
 * @dccps_ndp_count - number of Non Data Packets since last data packet
 * @dccps_mss_cache - current value of MSS (path MTU minus header sizes)
 * @dccps_rate_last - timestamp for rate-limiting DCCP-Sync (RFC 4340, 7.5.4)
 * @dccps_featneg - tracks feature-negotiation state (mostly during handshake)
 * @dccps_hc_rx_ackvec - rx half connection ack vector
 * @dccps_hc_rx_ccid - CCID used for the receiver (or receiving half-connection)
 * @dccps_hc_tx_ccid - CCID used for the sender (or sending half-connection)
 * @dccps_options_received - parsed set of retrieved options
 * @dccps_qpolicy - TX dequeueing policy, one of %dccp_packet_dequeueing_policy
 * @dccps_tx_qlen - maximum length of the TX queue
 * @dccps_role - role of this sock, one of %dccp_role
 * @dccps_hc_rx_insert_options - receiver wants to add options when acking
 * @dccps_hc_tx_insert_options - sender wants to add options when sending
 * @dccps_server_timewait - server holds timewait state on close (RFC 4340, 8.3)
 * @dccps_sync_scheduled - flag which signals "send out-of-band message soon"
 * @dccps_xmitlet - tasklet scheduled by the TX CCID to dequeue data packets
 * @dccps_xmit_timer - used by the TX CCID to delay sending (rate-based pacing)
 * @dccps_syn_rtt - RTT sample from Request/Response exchange (in usecs)
 */
struct dccp_sock {
        /* inet_connection_sock has to be the first member of dccp_sock */
        struct inet_connection_sock        dccps_inet_connection;
#define dccps_syn_rtt                        dccps_inet_connection.icsk_ack.lrcvtime
        __u64                                dccps_swl;
        __u64                                dccps_swh;
        __u64                                dccps_awl;
        __u64                                dccps_awh;
        __u64                                dccps_iss;
        __u64                                dccps_isr;
        __u64                                dccps_osr;
        __u64                                dccps_gss;
        __u64                                dccps_gsr;
        __u64                                dccps_gar;
        __be32                                dccps_service;
        __u32                                dccps_mss_cache;
        struct dccp_service_list        *dccps_service_list;
        __u32                                dccps_timestamp_echo;
        __u32                                dccps_timestamp_time;
        __u16                                dccps_l_ack_ratio;
        __u16                                dccps_r_ack_ratio;
        __u64                                dccps_l_seq_win:48;
        __u64                                dccps_r_seq_win:48;
        __u8                                dccps_pcslen:4;
        __u8                                dccps_pcrlen:4;
        __u8                                dccps_send_ndp_count:1;
        __u64                                dccps_ndp_count:48;
        unsigned long                        dccps_rate_last;
        struct list_head                dccps_featneg;
        struct dccp_ackvec                *dccps_hc_rx_ackvec;
        struct ccid                        *dccps_hc_rx_ccid;
        struct ccid                        *dccps_hc_tx_ccid;
        struct dccp_options_received        dccps_options_received;
        __u8                                dccps_qpolicy;
        __u32                                dccps_tx_qlen;
        enum dccp_role                        dccps_role:2;
        __u8                                dccps_hc_rx_insert_options:1;
        __u8                                dccps_hc_tx_insert_options:1;
        __u8                                dccps_server_timewait:1;
        __u8                                dccps_sync_scheduled:1;
        struct tasklet_struct                dccps_xmitlet;
        struct timer_list                dccps_xmit_timer;
};

#define dccp_sk(ptr)        container_of_const(ptr, struct dccp_sock, \
                                           dccps_inet_connection.icsk_inet.sk)

static inline const char *dccp_role(const struct sock *sk)
{
        switch (dccp_sk(sk)->dccps_role) {
        case DCCP_ROLE_UNDEFINED: return "undefined";
        case DCCP_ROLE_LISTEN:          return "listen";
        case DCCP_ROLE_SERVER:          return "server";
        case DCCP_ROLE_CLIENT:          return "client";
        }
        return NULL;
}

extern void dccp_syn_ack_timeout(const struct request_sock *req);

#endif /* _LINUX_DCCP_H */

























































    3 


    3 


    3 

    3 

    3 


    3 


























































































































































































































    8 




    8 
    8 


    8 











    8 



    8 
    8 



    8 





    3 













    3 


    3 
























    3 




    3 









    3 












    3 

    3 


    3 
    3 


    3 
    3 


    3 
    3 


    3 
    3 


    3 


    3 




















    4 

    4 


    4 


    4 
    4 






    4 
    4 

    4 



    3 









    3 


    3 




























    4 

    4 


    4 
    4 


























































































    4 

    4 
    4 

    4 




































    4 

    4 
    4 

    4 
















    4 










    4 


    4 






    4 






    3 
    4 

    4 
















    4 

    4 







    4 



























   10 

   10 


   10 










   10 
    9 




   10 
    9 



   10 


   10 



























   10 
   10 

   10 




































































































































    1 


































    1 











    1 


    1 

    1 
    1 

    1 

    1 











    1 






    1 
    1 

    1 


    1 







    1 





    1 


    1 



    1 
    1 









    1 



    1 







    1 

    1 





    1 
    1 

    1 

    1 





















































































































































































































































































































































































































































































































    3 
































































































































































































































































































































































































































































































































































































































































































































































    3 




    3 

    3 


    3 


    3 
    3 
    3 
    3 



    3 









    3 




    3 









































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   36 




   36 

   36 

   36 


   36 


























































































































































































































































































































































































































































































































































































































































































































































    1 



















    1 




    1 












    1 








    1 







    1 

    1 


    1 



    1 


    1 










    1 


    1 


    1 


    1 
    1 

    1 


    1 





























































































































































































































































































































   27 




















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2002-2005, Instant802 Networks, Inc.
 * Copyright 2005-2006, Devicescape Software, Inc.
 * Copyright 2006-2007        Jiri Benc <jbenc@suse.cz>
 * Copyright 2007        Johannes Berg <johannes@sipsolutions.net>
 * Copyright 2013-2014  Intel Mobile Communications GmbH
 * Copyright (C) 2015-2017        Intel Deutschland GmbH
 * Copyright (C) 2018-2024 Intel Corporation
 *
 * utilities for mac80211
 */

#include <net/mac80211.h>
#include <linux/netdevice.h>
#include <linux/export.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/skbuff.h>
#include <linux/etherdevice.h>
#include <linux/if_arp.h>
#include <linux/bitmap.h>
#include <linux/crc32.h>
#include <net/net_namespace.h>
#include <net/cfg80211.h>
#include <net/rtnetlink.h>
#include <kunit/visibility.h>

#include "ieee80211_i.h"
#include "driver-ops.h"
#include "rate.h"
#include "mesh.h"
#include "wme.h"
#include "led.h"
#include "wep.h"

/* privid for wiphys to determine whether they belong to us or not */
const void *const mac80211_wiphy_privid = &mac80211_wiphy_privid;

struct ieee80211_hw *wiphy_to_ieee80211_hw(struct wiphy *wiphy)
{
        struct ieee80211_local *local;

        local = wiphy_priv(wiphy);
        return &local->hw;
}
EXPORT_SYMBOL(wiphy_to_ieee80211_hw);

const struct ieee80211_conn_settings ieee80211_conn_settings_unlimited = {
        .mode = IEEE80211_CONN_MODE_EHT,
        .bw_limit = IEEE80211_CONN_BW_LIMIT_320,
};

u8 *ieee80211_get_bssid(struct ieee80211_hdr *hdr, size_t len,
                        enum nl80211_iftype type)
{
        __le16 fc = hdr->frame_control;

        if (ieee80211_is_data(fc)) {
                if (len < 24) /* drop incorrect hdr len (data) */
                        return NULL;

                if (ieee80211_has_a4(fc))
                        return NULL;
                if (ieee80211_has_tods(fc))
                        return hdr->addr1;
                if (ieee80211_has_fromds(fc))
                        return hdr->addr2;

                return hdr->addr3;
        }

        if (ieee80211_is_s1g_beacon(fc)) {
                struct ieee80211_ext *ext = (void *) hdr;

                return ext->u.s1g_beacon.sa;
        }

        if (ieee80211_is_mgmt(fc)) {
                if (len < 24) /* drop incorrect hdr len (mgmt) */
                        return NULL;
                return hdr->addr3;
        }

        if (ieee80211_is_ctl(fc)) {
                if (ieee80211_is_pspoll(fc))
                        return hdr->addr1;

                if (ieee80211_is_back_req(fc)) {
                        switch (type) {
                        case NL80211_IFTYPE_STATION:
                                return hdr->addr2;
                        case NL80211_IFTYPE_AP:
                        case NL80211_IFTYPE_AP_VLAN:
                                return hdr->addr1;
                        default:
                                break; /* fall through to the return */
                        }
                }
        }

        return NULL;
}
EXPORT_SYMBOL(ieee80211_get_bssid);

void ieee80211_tx_set_protected(struct ieee80211_tx_data *tx)
{
        struct sk_buff *skb;
        struct ieee80211_hdr *hdr;

        skb_queue_walk(&tx->skbs, skb) {
                hdr = (struct ieee80211_hdr *) skb->data;
                hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PROTECTED);
        }
}

int ieee80211_frame_duration(enum nl80211_band band, size_t len,
                             int rate, int erp, int short_preamble)
{
        int dur;

        /* calculate duration (in microseconds, rounded up to next higher
         * integer if it includes a fractional microsecond) to send frame of
         * len bytes (does not include FCS) at the given rate. Duration will
         * also include SIFS.
         *
         * rate is in 100 kbps, so divident is multiplied by 10 in the
         * DIV_ROUND_UP() operations.
         */

        if (band == NL80211_BAND_5GHZ || erp) {
                /*
                 * OFDM:
                 *
                 * N_DBPS = DATARATE x 4
                 * N_SYM = Ceiling((16+8xLENGTH+6) / N_DBPS)
                 *        (16 = SIGNAL time, 6 = tail bits)
                 * TXTIME = T_PREAMBLE + T_SIGNAL + T_SYM x N_SYM + Signal Ext
                 *
                 * T_SYM = 4 usec
                 * 802.11a - 18.5.2: aSIFSTime = 16 usec
                 * 802.11g - 19.8.4: aSIFSTime = 10 usec +
                 *        signal ext = 6 usec
                 */
                dur = 16; /* SIFS + signal ext */
                dur += 16; /* IEEE 802.11-2012 18.3.2.4: T_PREAMBLE = 16 usec */
                dur += 4; /* IEEE 802.11-2012 18.3.2.4: T_SIGNAL = 4 usec */

                /* rates should already consider the channel bandwidth,
                 * don't apply divisor again.
                 */
                dur += 4 * DIV_ROUND_UP((16 + 8 * (len + 4) + 6) * 10,
                                        4 * rate); /* T_SYM x N_SYM */
        } else {
                /*
                 * 802.11b or 802.11g with 802.11b compatibility:
                 * 18.3.4: TXTIME = PreambleLength + PLCPHeaderTime +
                 * Ceiling(((LENGTH+PBCC)x8)/DATARATE). PBCC=0.
                 *
                 * 802.11 (DS): 15.3.3, 802.11b: 18.3.4
                 * aSIFSTime = 10 usec
                 * aPreambleLength = 144 usec or 72 usec with short preamble
                 * aPLCPHeaderLength = 48 usec or 24 usec with short preamble
                 */
                dur = 10; /* aSIFSTime = 10 usec */
                dur += short_preamble ? (72 + 24) : (144 + 48);

                dur += DIV_ROUND_UP(8 * (len + 4) * 10, rate);
        }

        return dur;
}

/* Exported duration function for driver use */
__le16 ieee80211_generic_frame_duration(struct ieee80211_hw *hw,
                                        struct ieee80211_vif *vif,
                                        enum nl80211_band band,
                                        size_t frame_len,
                                        struct ieee80211_rate *rate)
{
        struct ieee80211_sub_if_data *sdata;
        u16 dur;
        int erp;
        bool short_preamble = false;

        erp = 0;
        if (vif) {
                sdata = vif_to_sdata(vif);
                short_preamble = sdata->vif.bss_conf.use_short_preamble;
                if (sdata->deflink.operating_11g_mode)
                        erp = rate->flags & IEEE80211_RATE_ERP_G;
        }

        dur = ieee80211_frame_duration(band, frame_len, rate->bitrate, erp,
                                       short_preamble);

        return cpu_to_le16(dur);
}
EXPORT_SYMBOL(ieee80211_generic_frame_duration);

__le16 ieee80211_rts_duration(struct ieee80211_hw *hw,
                              struct ieee80211_vif *vif, size_t frame_len,
                              const struct ieee80211_tx_info *frame_txctl)
{
        struct ieee80211_local *local = hw_to_local(hw);
        struct ieee80211_rate *rate;
        struct ieee80211_sub_if_data *sdata;
        bool short_preamble;
        int erp, bitrate;
        u16 dur;
        struct ieee80211_supported_band *sband;

        sband = local->hw.wiphy->bands[frame_txctl->band];

        short_preamble = false;

        rate = &sband->bitrates[frame_txctl->control.rts_cts_rate_idx];

        erp = 0;
        if (vif) {
                sdata = vif_to_sdata(vif);
                short_preamble = sdata->vif.bss_conf.use_short_preamble;
                if (sdata->deflink.operating_11g_mode)
                        erp = rate->flags & IEEE80211_RATE_ERP_G;
        }

        bitrate = rate->bitrate;

        /* CTS duration */
        dur = ieee80211_frame_duration(sband->band, 10, bitrate,
                                       erp, short_preamble);
        /* Data frame duration */
        dur += ieee80211_frame_duration(sband->band, frame_len, bitrate,
                                        erp, short_preamble);
        /* ACK duration */
        dur += ieee80211_frame_duration(sband->band, 10, bitrate,
                                        erp, short_preamble);

        return cpu_to_le16(dur);
}
EXPORT_SYMBOL(ieee80211_rts_duration);

__le16 ieee80211_ctstoself_duration(struct ieee80211_hw *hw,
                                    struct ieee80211_vif *vif,
                                    size_t frame_len,
                                    const struct ieee80211_tx_info *frame_txctl)
{
        struct ieee80211_local *local = hw_to_local(hw);
        struct ieee80211_rate *rate;
        struct ieee80211_sub_if_data *sdata;
        bool short_preamble;
        int erp, bitrate;
        u16 dur;
        struct ieee80211_supported_band *sband;

        sband = local->hw.wiphy->bands[frame_txctl->band];

        short_preamble = false;

        rate = &sband->bitrates[frame_txctl->control.rts_cts_rate_idx];
        erp = 0;
        if (vif) {
                sdata = vif_to_sdata(vif);
                short_preamble = sdata->vif.bss_conf.use_short_preamble;
                if (sdata->deflink.operating_11g_mode)
                        erp = rate->flags & IEEE80211_RATE_ERP_G;
        }

        bitrate = rate->bitrate;

        /* Data frame duration */
        dur = ieee80211_frame_duration(sband->band, frame_len, bitrate,
                                       erp, short_preamble);
        if (!(frame_txctl->flags & IEEE80211_TX_CTL_NO_ACK)) {
                /* ACK duration */
                dur += ieee80211_frame_duration(sband->band, 10, bitrate,
                                                erp, short_preamble);
        }

        return cpu_to_le16(dur);
}
EXPORT_SYMBOL(ieee80211_ctstoself_duration);

static void wake_tx_push_queue(struct ieee80211_local *local,
                               struct ieee80211_sub_if_data *sdata,
                               struct ieee80211_txq *queue)
{
        struct ieee80211_tx_control control = {
                .sta = queue->sta,
        };
        struct sk_buff *skb;

        while (1) {
                skb = ieee80211_tx_dequeue(&local->hw, queue);
                if (!skb)
                        break;

                drv_tx(local, &control, skb);
        }
}

/* wake_tx_queue handler for driver not implementing a custom one*/
void ieee80211_handle_wake_tx_queue(struct ieee80211_hw *hw,
                                    struct ieee80211_txq *txq)
{
        struct ieee80211_local *local = hw_to_local(hw);
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(txq->vif);
        struct ieee80211_txq *queue;

        spin_lock(&local->handle_wake_tx_queue_lock);

        /* Use ieee80211_next_txq() for airtime fairness accounting */
        ieee80211_txq_schedule_start(hw, txq->ac);
        while ((queue = ieee80211_next_txq(hw, txq->ac))) {
                wake_tx_push_queue(local, sdata, queue);
                ieee80211_return_txq(hw, queue, false);
        }
        ieee80211_txq_schedule_end(hw, txq->ac);
        spin_unlock(&local->handle_wake_tx_queue_lock);
}
EXPORT_SYMBOL(ieee80211_handle_wake_tx_queue);

static void __ieee80211_wake_txqs(struct ieee80211_sub_if_data *sdata, int ac)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_vif *vif = &sdata->vif;
        struct fq *fq = &local->fq;
        struct ps_data *ps = NULL;
        struct txq_info *txqi;
        struct sta_info *sta;
        int i;

        local_bh_disable();
        spin_lock(&fq->lock);

        if (!test_bit(SDATA_STATE_RUNNING, &sdata->state))
                goto out;

        if (sdata->vif.type == NL80211_IFTYPE_AP)
                ps = &sdata->bss->ps;

        list_for_each_entry_rcu(sta, &local->sta_list, list) {
                if (sdata != sta->sdata)
                        continue;

                for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
                        struct ieee80211_txq *txq = sta->sta.txq[i];

                        if (!txq)
                                continue;

                        txqi = to_txq_info(txq);

                        if (ac != txq->ac)
                                continue;

                        if (!test_and_clear_bit(IEEE80211_TXQ_DIRTY,
                                                &txqi->flags))
                                continue;

                        spin_unlock(&fq->lock);
                        drv_wake_tx_queue(local, txqi);
                        spin_lock(&fq->lock);
                }
        }

        if (!vif->txq)
                goto out;

        txqi = to_txq_info(vif->txq);

        if (!test_and_clear_bit(IEEE80211_TXQ_DIRTY, &txqi->flags) ||
            (ps && atomic_read(&ps->num_sta_ps)) || ac != vif->txq->ac)
                goto out;

        spin_unlock(&fq->lock);

        drv_wake_tx_queue(local, txqi);
        local_bh_enable();
        return;
out:
        spin_unlock(&fq->lock);
        local_bh_enable();
}

static void
__releases(&local->queue_stop_reason_lock)
__acquires(&local->queue_stop_reason_lock)
_ieee80211_wake_txqs(struct ieee80211_local *local, unsigned long *flags)
{
        struct ieee80211_sub_if_data *sdata;
        int n_acs = IEEE80211_NUM_ACS;
        int i;

        rcu_read_lock();

        if (local->hw.queues < IEEE80211_NUM_ACS)
                n_acs = 1;

        for (i = 0; i < local->hw.queues; i++) {
                if (local->queue_stop_reasons[i])
                        continue;

                spin_unlock_irqrestore(&local->queue_stop_reason_lock, *flags);
                list_for_each_entry_rcu(sdata, &local->interfaces, list) {
                        int ac;

                        for (ac = 0; ac < n_acs; ac++) {
                                int ac_queue = sdata->vif.hw_queue[ac];

                                if (ac_queue == i ||
                                    sdata->vif.cab_queue == i)
                                        __ieee80211_wake_txqs(sdata, ac);
                        }
                }
                spin_lock_irqsave(&local->queue_stop_reason_lock, *flags);
        }

        rcu_read_unlock();
}

void ieee80211_wake_txqs(struct tasklet_struct *t)
{
        struct ieee80211_local *local = from_tasklet(local, t,
                                                     wake_txqs_tasklet);
        unsigned long flags;

        spin_lock_irqsave(&local->queue_stop_reason_lock, flags);
        _ieee80211_wake_txqs(local, &flags);
        spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
}

static void __ieee80211_wake_queue(struct ieee80211_hw *hw, int queue,
                                   enum queue_stop_reason reason,
                                   bool refcounted,
                                   unsigned long *flags)
{
        struct ieee80211_local *local = hw_to_local(hw);

        trace_wake_queue(local, queue, reason);

        if (WARN_ON(queue >= hw->queues))
                return;

        if (!test_bit(reason, &local->queue_stop_reasons[queue]))
                return;

        if (!refcounted) {
                local->q_stop_reasons[queue][reason] = 0;
        } else {
                local->q_stop_reasons[queue][reason]--;
                if (WARN_ON(local->q_stop_reasons[queue][reason] < 0))
                        local->q_stop_reasons[queue][reason] = 0;
        }

        if (local->q_stop_reasons[queue][reason] == 0)
                __clear_bit(reason, &local->queue_stop_reasons[queue]);

        if (local->queue_stop_reasons[queue] != 0)
                /* someone still has this queue stopped */
                return;

        if (!skb_queue_empty(&local->pending[queue]))
                tasklet_schedule(&local->tx_pending_tasklet);

        /*
         * Calling _ieee80211_wake_txqs here can be a problem because it may
         * release queue_stop_reason_lock which has been taken by
         * __ieee80211_wake_queue's caller. It is certainly not very nice to
         * release someone's lock, but it is fine because all the callers of
         * __ieee80211_wake_queue call it right before releasing the lock.
         */
        if (reason == IEEE80211_QUEUE_STOP_REASON_DRIVER)
                tasklet_schedule(&local->wake_txqs_tasklet);
        else
                _ieee80211_wake_txqs(local, flags);
}

void ieee80211_wake_queue_by_reason(struct ieee80211_hw *hw, int queue,
                                    enum queue_stop_reason reason,
                                    bool refcounted)
{
        struct ieee80211_local *local = hw_to_local(hw);
        unsigned long flags;

        spin_lock_irqsave(&local->queue_stop_reason_lock, flags);
        __ieee80211_wake_queue(hw, queue, reason, refcounted, &flags);
        spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
}

void ieee80211_wake_queue(struct ieee80211_hw *hw, int queue)
{
        ieee80211_wake_queue_by_reason(hw, queue,
                                       IEEE80211_QUEUE_STOP_REASON_DRIVER,
                                       false);
}
EXPORT_SYMBOL(ieee80211_wake_queue);

static void __ieee80211_stop_queue(struct ieee80211_hw *hw, int queue,
                                   enum queue_stop_reason reason,
                                   bool refcounted)
{
        struct ieee80211_local *local = hw_to_local(hw);

        trace_stop_queue(local, queue, reason);

        if (WARN_ON(queue >= hw->queues))
                return;

        if (!refcounted)
                local->q_stop_reasons[queue][reason] = 1;
        else
                local->q_stop_reasons[queue][reason]++;

        set_bit(reason, &local->queue_stop_reasons[queue]);
}

void ieee80211_stop_queue_by_reason(struct ieee80211_hw *hw, int queue,
                                    enum queue_stop_reason reason,
                                    bool refcounted)
{
        struct ieee80211_local *local = hw_to_local(hw);
        unsigned long flags;

        spin_lock_irqsave(&local->queue_stop_reason_lock, flags);
        __ieee80211_stop_queue(hw, queue, reason, refcounted);
        spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
}

void ieee80211_stop_queue(struct ieee80211_hw *hw, int queue)
{
        ieee80211_stop_queue_by_reason(hw, queue,
                                       IEEE80211_QUEUE_STOP_REASON_DRIVER,
                                       false);
}
EXPORT_SYMBOL(ieee80211_stop_queue);

void ieee80211_add_pending_skb(struct ieee80211_local *local,
                               struct sk_buff *skb)
{
        struct ieee80211_hw *hw = &local->hw;
        unsigned long flags;
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
        int queue = info->hw_queue;

        if (WARN_ON(!info->control.vif)) {
                ieee80211_free_txskb(&local->hw, skb);
                return;
        }

        spin_lock_irqsave(&local->queue_stop_reason_lock, flags);
        __ieee80211_stop_queue(hw, queue, IEEE80211_QUEUE_STOP_REASON_SKB_ADD,
                               false);
        __skb_queue_tail(&local->pending[queue], skb);
        __ieee80211_wake_queue(hw, queue, IEEE80211_QUEUE_STOP_REASON_SKB_ADD,
                               false, &flags);
        spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
}

void ieee80211_add_pending_skbs(struct ieee80211_local *local,
                                struct sk_buff_head *skbs)
{
        struct ieee80211_hw *hw = &local->hw;
        struct sk_buff *skb;
        unsigned long flags;
        int queue, i;

        spin_lock_irqsave(&local->queue_stop_reason_lock, flags);
        while ((skb = skb_dequeue(skbs))) {
                struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);

                if (WARN_ON(!info->control.vif)) {
                        ieee80211_free_txskb(&local->hw, skb);
                        continue;
                }

                queue = info->hw_queue;

                __ieee80211_stop_queue(hw, queue,
                                IEEE80211_QUEUE_STOP_REASON_SKB_ADD,
                                false);

                __skb_queue_tail(&local->pending[queue], skb);
        }

        for (i = 0; i < hw->queues; i++)
                __ieee80211_wake_queue(hw, i,
                        IEEE80211_QUEUE_STOP_REASON_SKB_ADD,
                        false, &flags);
        spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
}

void ieee80211_stop_queues_by_reason(struct ieee80211_hw *hw,
                                     unsigned long queues,
                                     enum queue_stop_reason reason,
                                     bool refcounted)
{
        struct ieee80211_local *local = hw_to_local(hw);
        unsigned long flags;
        int i;

        spin_lock_irqsave(&local->queue_stop_reason_lock, flags);

        for_each_set_bit(i, &queues, hw->queues)
                __ieee80211_stop_queue(hw, i, reason, refcounted);

        spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
}

void ieee80211_stop_queues(struct ieee80211_hw *hw)
{
        ieee80211_stop_queues_by_reason(hw, IEEE80211_MAX_QUEUE_MAP,
                                        IEEE80211_QUEUE_STOP_REASON_DRIVER,
                                        false);
}
EXPORT_SYMBOL(ieee80211_stop_queues);

int ieee80211_queue_stopped(struct ieee80211_hw *hw, int queue)
{
        struct ieee80211_local *local = hw_to_local(hw);
        unsigned long flags;
        int ret;

        if (WARN_ON(queue >= hw->queues))
                return true;

        spin_lock_irqsave(&local->queue_stop_reason_lock, flags);
        ret = test_bit(IEEE80211_QUEUE_STOP_REASON_DRIVER,
                       &local->queue_stop_reasons[queue]);
        spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
        return ret;
}
EXPORT_SYMBOL(ieee80211_queue_stopped);

void ieee80211_wake_queues_by_reason(struct ieee80211_hw *hw,
                                     unsigned long queues,
                                     enum queue_stop_reason reason,
                                     bool refcounted)
{
        struct ieee80211_local *local = hw_to_local(hw);
        unsigned long flags;
        int i;

        spin_lock_irqsave(&local->queue_stop_reason_lock, flags);

        for_each_set_bit(i, &queues, hw->queues)
                __ieee80211_wake_queue(hw, i, reason, refcounted, &flags);

        spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
}

void ieee80211_wake_queues(struct ieee80211_hw *hw)
{
        ieee80211_wake_queues_by_reason(hw, IEEE80211_MAX_QUEUE_MAP,
                                        IEEE80211_QUEUE_STOP_REASON_DRIVER,
                                        false);
}
EXPORT_SYMBOL(ieee80211_wake_queues);

static unsigned int
ieee80211_get_vif_queues(struct ieee80211_local *local,
                         struct ieee80211_sub_if_data *sdata)
{
        unsigned int queues;

        if (sdata && ieee80211_hw_check(&local->hw, QUEUE_CONTROL)) {
                int ac;

                queues = 0;

                for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
                        queues |= BIT(sdata->vif.hw_queue[ac]);
                if (sdata->vif.cab_queue != IEEE80211_INVAL_HW_QUEUE)
                        queues |= BIT(sdata->vif.cab_queue);
        } else {
                /* all queues */
                queues = BIT(local->hw.queues) - 1;
        }

        return queues;
}

void __ieee80211_flush_queues(struct ieee80211_local *local,
                              struct ieee80211_sub_if_data *sdata,
                              unsigned int queues, bool drop)
{
        if (!local->ops->flush)
                return;

        /*
         * If no queue was set, or if the HW doesn't support
         * IEEE80211_HW_QUEUE_CONTROL - flush all queues
         */
        if (!queues || !ieee80211_hw_check(&local->hw, QUEUE_CONTROL))
                queues = ieee80211_get_vif_queues(local, sdata);

        ieee80211_stop_queues_by_reason(&local->hw, queues,
                                        IEEE80211_QUEUE_STOP_REASON_FLUSH,
                                        false);

        if (drop) {
                struct sta_info *sta;

                /* Purge the queues, so the frames on them won't be
                 * sent during __ieee80211_wake_queue()
                 */
                list_for_each_entry(sta, &local->sta_list, list) {
                        if (sdata != sta->sdata)
                                continue;
                        ieee80211_purge_sta_txqs(sta);
                }
        }

        drv_flush(local, sdata, queues, drop);

        ieee80211_wake_queues_by_reason(&local->hw, queues,
                                        IEEE80211_QUEUE_STOP_REASON_FLUSH,
                                        false);
}

void ieee80211_flush_queues(struct ieee80211_local *local,
                            struct ieee80211_sub_if_data *sdata, bool drop)
{
        __ieee80211_flush_queues(local, sdata, 0, drop);
}

void ieee80211_stop_vif_queues(struct ieee80211_local *local,
                               struct ieee80211_sub_if_data *sdata,
                               enum queue_stop_reason reason)
{
        ieee80211_stop_queues_by_reason(&local->hw,
                                        ieee80211_get_vif_queues(local, sdata),
                                        reason, true);
}

void ieee80211_wake_vif_queues(struct ieee80211_local *local,
                               struct ieee80211_sub_if_data *sdata,
                               enum queue_stop_reason reason)
{
        ieee80211_wake_queues_by_reason(&local->hw,
                                        ieee80211_get_vif_queues(local, sdata),
                                        reason, true);
}

static void __iterate_interfaces(struct ieee80211_local *local,
                                 u32 iter_flags,
                                 void (*iterator)(void *data, u8 *mac,
                                                  struct ieee80211_vif *vif),
                                 void *data)
{
        struct ieee80211_sub_if_data *sdata;
        bool active_only = iter_flags & IEEE80211_IFACE_ITER_ACTIVE;

        list_for_each_entry_rcu(sdata, &local->interfaces, list,
                                lockdep_is_held(&local->iflist_mtx) ||
                                lockdep_is_held(&local->hw.wiphy->mtx)) {
                switch (sdata->vif.type) {
                case NL80211_IFTYPE_MONITOR:
                        if (!(sdata->u.mntr.flags & MONITOR_FLAG_ACTIVE) &&
                            !ieee80211_hw_check(&local->hw, NO_VIRTUAL_MONITOR))
                                continue;
                        break;
                case NL80211_IFTYPE_AP_VLAN:
                        continue;
                default:
                        break;
                }
                if (!(iter_flags & IEEE80211_IFACE_ITER_RESUME_ALL) &&
                    active_only && !(sdata->flags & IEEE80211_SDATA_IN_DRIVER))
                        continue;
                if ((iter_flags & IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER) &&
                    !(sdata->flags & IEEE80211_SDATA_IN_DRIVER))
                        continue;
                if (ieee80211_sdata_running(sdata) || !active_only)
                        iterator(data, sdata->vif.addr,
                                 &sdata->vif);
        }

        sdata = rcu_dereference_check(local->monitor_sdata,
                                      lockdep_is_held(&local->iflist_mtx) ||
                                      lockdep_is_held(&local->hw.wiphy->mtx));
        if (sdata && ieee80211_hw_check(&local->hw, WANT_MONITOR_VIF) &&
            (iter_flags & IEEE80211_IFACE_ITER_RESUME_ALL || !active_only ||
             sdata->flags & IEEE80211_SDATA_IN_DRIVER))
                iterator(data, sdata->vif.addr, &sdata->vif);
}

void ieee80211_iterate_interfaces(
        struct ieee80211_hw *hw, u32 iter_flags,
        void (*iterator)(void *data, u8 *mac,
                         struct ieee80211_vif *vif),
        void *data)
{
        struct ieee80211_local *local = hw_to_local(hw);

        mutex_lock(&local->iflist_mtx);
        __iterate_interfaces(local, iter_flags, iterator, data);
        mutex_unlock(&local->iflist_mtx);
}
EXPORT_SYMBOL_GPL(ieee80211_iterate_interfaces);

void ieee80211_iterate_active_interfaces_atomic(
        struct ieee80211_hw *hw, u32 iter_flags,
        void (*iterator)(void *data, u8 *mac,
                         struct ieee80211_vif *vif),
        void *data)
{
        struct ieee80211_local *local = hw_to_local(hw);

        rcu_read_lock();
        __iterate_interfaces(local, iter_flags | IEEE80211_IFACE_ITER_ACTIVE,
                             iterator, data);
        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(ieee80211_iterate_active_interfaces_atomic);

void ieee80211_iterate_active_interfaces_mtx(
        struct ieee80211_hw *hw, u32 iter_flags,
        void (*iterator)(void *data, u8 *mac,
                         struct ieee80211_vif *vif),
        void *data)
{
        struct ieee80211_local *local = hw_to_local(hw);

        lockdep_assert_wiphy(hw->wiphy);

        __iterate_interfaces(local, iter_flags | IEEE80211_IFACE_ITER_ACTIVE,
                             iterator, data);
}
EXPORT_SYMBOL_GPL(ieee80211_iterate_active_interfaces_mtx);

static void __iterate_stations(struct ieee80211_local *local,
                               void (*iterator)(void *data,
                                                struct ieee80211_sta *sta),
                               void *data)
{
        struct sta_info *sta;

        list_for_each_entry_rcu(sta, &local->sta_list, list,
                                lockdep_is_held(&local->hw.wiphy->mtx)) {
                if (!sta->uploaded)
                        continue;

                iterator(data, &sta->sta);
        }
}

void ieee80211_iterate_stations_atomic(struct ieee80211_hw *hw,
                        void (*iterator)(void *data,
                                         struct ieee80211_sta *sta),
                        void *data)
{
        struct ieee80211_local *local = hw_to_local(hw);

        rcu_read_lock();
        __iterate_stations(local, iterator, data);
        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(ieee80211_iterate_stations_atomic);

void ieee80211_iterate_stations_mtx(struct ieee80211_hw *hw,
                                    void (*iterator)(void *data,
                                                     struct ieee80211_sta *sta),
                                    void *data)
{
        struct ieee80211_local *local = hw_to_local(hw);

        lockdep_assert_wiphy(local->hw.wiphy);

        __iterate_stations(local, iterator, data);
}
EXPORT_SYMBOL_GPL(ieee80211_iterate_stations_mtx);

struct ieee80211_vif *wdev_to_ieee80211_vif(struct wireless_dev *wdev)
{
        struct ieee80211_sub_if_data *sdata = IEEE80211_WDEV_TO_SUB_IF(wdev);

        if (!ieee80211_sdata_running(sdata) ||
            !(sdata->flags & IEEE80211_SDATA_IN_DRIVER))
                return NULL;
        return &sdata->vif;
}
EXPORT_SYMBOL_GPL(wdev_to_ieee80211_vif);

struct wireless_dev *ieee80211_vif_to_wdev(struct ieee80211_vif *vif)
{
        if (!vif)
                return NULL;

        return &vif_to_sdata(vif)->wdev;
}
EXPORT_SYMBOL_GPL(ieee80211_vif_to_wdev);

/*
 * Nothing should have been stuffed into the workqueue during
 * the suspend->resume cycle. Since we can't check each caller
 * of this function if we are already quiescing / suspended,
 * check here and don't WARN since this can actually happen when
 * the rx path (for example) is racing against __ieee80211_suspend
 * and suspending / quiescing was set after the rx path checked
 * them.
 */
static bool ieee80211_can_queue_work(struct ieee80211_local *local)
{
        if (local->quiescing || (local->suspended && !local->resuming)) {
                pr_warn("queueing ieee80211 work while going to suspend\n");
                return false;
        }

        return true;
}

void ieee80211_queue_work(struct ieee80211_hw *hw, struct work_struct *work)
{
        struct ieee80211_local *local = hw_to_local(hw);

        if (!ieee80211_can_queue_work(local))
                return;

        queue_work(local->workqueue, work);
}
EXPORT_SYMBOL(ieee80211_queue_work);

void ieee80211_queue_delayed_work(struct ieee80211_hw *hw,
                                  struct delayed_work *dwork,
                                  unsigned long delay)
{
        struct ieee80211_local *local = hw_to_local(hw);

        if (!ieee80211_can_queue_work(local))
                return;

        queue_delayed_work(local->workqueue, dwork, delay);
}
EXPORT_SYMBOL(ieee80211_queue_delayed_work);

void ieee80211_regulatory_limit_wmm_params(struct ieee80211_sub_if_data *sdata,
                                           struct ieee80211_tx_queue_params
                                           *qparam, int ac)
{
        struct ieee80211_chanctx_conf *chanctx_conf;
        const struct ieee80211_reg_rule *rrule;
        const struct ieee80211_wmm_ac *wmm_ac;
        u16 center_freq = 0;

        if (sdata->vif.type != NL80211_IFTYPE_AP &&
            sdata->vif.type != NL80211_IFTYPE_STATION)
                return;

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
        if (chanctx_conf)
                center_freq = chanctx_conf->def.chan->center_freq;

        if (!center_freq) {
                rcu_read_unlock();
                return;
        }

        rrule = freq_reg_info(sdata->wdev.wiphy, MHZ_TO_KHZ(center_freq));

        if (IS_ERR_OR_NULL(rrule) || !rrule->has_wmm) {
                rcu_read_unlock();
                return;
        }

        if (sdata->vif.type == NL80211_IFTYPE_AP)
                wmm_ac = &rrule->wmm_rule.ap[ac];
        else
                wmm_ac = &rrule->wmm_rule.client[ac];
        qparam->cw_min = max_t(u16, qparam->cw_min, wmm_ac->cw_min);
        qparam->cw_max = max_t(u16, qparam->cw_max, wmm_ac->cw_max);
        qparam->aifs = max_t(u8, qparam->aifs, wmm_ac->aifsn);
        qparam->txop = min_t(u16, qparam->txop, wmm_ac->cot / 32);
        rcu_read_unlock();
}

void ieee80211_set_wmm_default(struct ieee80211_link_data *link,
                               bool bss_notify, bool enable_qos)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_tx_queue_params qparam;
        struct ieee80211_chanctx_conf *chanctx_conf;
        int ac;
        bool use_11b;
        bool is_ocb; /* Use another EDCA parameters if dot11OCBActivated=true */
        int aCWmin, aCWmax;

        if (!local->ops->conf_tx)
                return;

        if (local->hw.queues < IEEE80211_NUM_ACS)
                return;

        memset(&qparam, 0, sizeof(qparam));

        rcu_read_lock();
        chanctx_conf = rcu_dereference(link->conf->chanctx_conf);
        use_11b = (chanctx_conf &&
                   chanctx_conf->def.chan->band == NL80211_BAND_2GHZ) &&
                 !link->operating_11g_mode;
        rcu_read_unlock();

        is_ocb = (sdata->vif.type == NL80211_IFTYPE_OCB);

        /* Set defaults according to 802.11-2007 Table 7-37 */
        aCWmax = 1023;
        if (use_11b)
                aCWmin = 31;
        else
                aCWmin = 15;

        /* Configure old 802.11b/g medium access rules. */
        qparam.cw_max = aCWmax;
        qparam.cw_min = aCWmin;
        qparam.txop = 0;
        qparam.aifs = 2;

        for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
                /* Update if QoS is enabled. */
                if (enable_qos) {
                        switch (ac) {
                        case IEEE80211_AC_BK:
                                qparam.cw_max = aCWmax;
                                qparam.cw_min = aCWmin;
                                qparam.txop = 0;
                                if (is_ocb)
                                        qparam.aifs = 9;
                                else
                                        qparam.aifs = 7;
                                break;
                        /* never happens but let's not leave undefined */
                        default:
                        case IEEE80211_AC_BE:
                                qparam.cw_max = aCWmax;
                                qparam.cw_min = aCWmin;
                                qparam.txop = 0;
                                if (is_ocb)
                                        qparam.aifs = 6;
                                else
                                        qparam.aifs = 3;
                                break;
                        case IEEE80211_AC_VI:
                                qparam.cw_max = aCWmin;
                                qparam.cw_min = (aCWmin + 1) / 2 - 1;
                                if (is_ocb)
                                        qparam.txop = 0;
                                else if (use_11b)
                                        qparam.txop = 6016/32;
                                else
                                        qparam.txop = 3008/32;

                                if (is_ocb)
                                        qparam.aifs = 3;
                                else
                                        qparam.aifs = 2;
                                break;
                        case IEEE80211_AC_VO:
                                qparam.cw_max = (aCWmin + 1) / 2 - 1;
                                qparam.cw_min = (aCWmin + 1) / 4 - 1;
                                if (is_ocb)
                                        qparam.txop = 0;
                                else if (use_11b)
                                        qparam.txop = 3264/32;
                                else
                                        qparam.txop = 1504/32;
                                qparam.aifs = 2;
                                break;
                        }
                }
                ieee80211_regulatory_limit_wmm_params(sdata, &qparam, ac);

                qparam.uapsd = false;

                link->tx_conf[ac] = qparam;
                drv_conf_tx(local, link, ac, &qparam);
        }

        if (sdata->vif.type != NL80211_IFTYPE_MONITOR &&
            sdata->vif.type != NL80211_IFTYPE_P2P_DEVICE &&
            sdata->vif.type != NL80211_IFTYPE_NAN) {
                link->conf->qos = enable_qos;
                if (bss_notify)
                        ieee80211_link_info_change_notify(sdata, link,
                                                          BSS_CHANGED_QOS);
        }
}

void ieee80211_send_auth(struct ieee80211_sub_if_data *sdata,
                         u16 transaction, u16 auth_alg, u16 status,
                         const u8 *extra, size_t extra_len, const u8 *da,
                         const u8 *bssid, const u8 *key, u8 key_len, u8 key_idx,
                         u32 tx_flags)
{
        struct ieee80211_local *local = sdata->local;
        struct sk_buff *skb;
        struct ieee80211_mgmt *mgmt;
        bool multi_link = ieee80211_vif_is_mld(&sdata->vif);
        struct {
                u8 id;
                u8 len;
                u8 ext_id;
                struct ieee80211_multi_link_elem ml;
                struct ieee80211_mle_basic_common_info basic;
        } __packed mle = {
                .id = WLAN_EID_EXTENSION,
                .len = sizeof(mle) - 2,
                .ext_id = WLAN_EID_EXT_EHT_MULTI_LINK,
                .ml.control = cpu_to_le16(IEEE80211_ML_CONTROL_TYPE_BASIC),
                .basic.len = sizeof(mle.basic),
        };
        int err;

        memcpy(mle.basic.mld_mac_addr, sdata->vif.addr, ETH_ALEN);

        /* 24 + 6 = header + auth_algo + auth_transaction + status_code */
        skb = dev_alloc_skb(local->hw.extra_tx_headroom + IEEE80211_WEP_IV_LEN +
                            24 + 6 + extra_len + IEEE80211_WEP_ICV_LEN +
                            multi_link * sizeof(mle));
        if (!skb)
                return;

        skb_reserve(skb, local->hw.extra_tx_headroom + IEEE80211_WEP_IV_LEN);

        mgmt = skb_put_zero(skb, 24 + 6);
        mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
                                          IEEE80211_STYPE_AUTH);
        memcpy(mgmt->da, da, ETH_ALEN);
        memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN);
        memcpy(mgmt->bssid, bssid, ETH_ALEN);
        mgmt->u.auth.auth_alg = cpu_to_le16(auth_alg);
        mgmt->u.auth.auth_transaction = cpu_to_le16(transaction);
        mgmt->u.auth.status_code = cpu_to_le16(status);
        if (extra)
                skb_put_data(skb, extra, extra_len);
        if (multi_link)
                skb_put_data(skb, &mle, sizeof(mle));

        if (auth_alg == WLAN_AUTH_SHARED_KEY && transaction == 3) {
                mgmt->frame_control |= cpu_to_le16(IEEE80211_FCTL_PROTECTED);
                err = ieee80211_wep_encrypt(local, skb, key, key_len, key_idx);
                if (WARN_ON(err)) {
                        kfree_skb(skb);
                        return;
                }
        }

        IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT |
                                        tx_flags;
        ieee80211_tx_skb(sdata, skb);
}

void ieee80211_send_deauth_disassoc(struct ieee80211_sub_if_data *sdata,
                                    const u8 *da, const u8 *bssid,
                                    u16 stype, u16 reason,
                                    bool send_frame, u8 *frame_buf)
{
        struct ieee80211_local *local = sdata->local;
        struct sk_buff *skb;
        struct ieee80211_mgmt *mgmt = (void *)frame_buf;

        /* build frame */
        mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT | stype);
        mgmt->duration = 0; /* initialize only */
        mgmt->seq_ctrl = 0; /* initialize only */
        memcpy(mgmt->da, da, ETH_ALEN);
        memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN);
        memcpy(mgmt->bssid, bssid, ETH_ALEN);
        /* u.deauth.reason_code == u.disassoc.reason_code */
        mgmt->u.deauth.reason_code = cpu_to_le16(reason);

        if (send_frame) {
                skb = dev_alloc_skb(local->hw.extra_tx_headroom +
                                    IEEE80211_DEAUTH_FRAME_LEN);
                if (!skb)
                        return;

                skb_reserve(skb, local->hw.extra_tx_headroom);

                /* copy in frame */
                skb_put_data(skb, mgmt, IEEE80211_DEAUTH_FRAME_LEN);

                if (sdata->vif.type != NL80211_IFTYPE_STATION ||
                    !(sdata->u.mgd.flags & IEEE80211_STA_MFP_ENABLED))
                        IEEE80211_SKB_CB(skb)->flags |=
                                IEEE80211_TX_INTFL_DONT_ENCRYPT;

                ieee80211_tx_skb(sdata, skb);
        }
}

static int ieee80211_put_s1g_cap(struct sk_buff *skb,
                                 struct ieee80211_sta_s1g_cap *s1g_cap)
{
        if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_s1g_cap))
                return -ENOBUFS;

        skb_put_u8(skb, WLAN_EID_S1G_CAPABILITIES);
        skb_put_u8(skb, sizeof(struct ieee80211_s1g_cap));

        skb_put_data(skb, &s1g_cap->cap, sizeof(s1g_cap->cap));
        skb_put_data(skb, &s1g_cap->nss_mcs, sizeof(s1g_cap->nss_mcs));

        return 0;
}

static int ieee80211_put_preq_ies_band(struct sk_buff *skb,
                                       struct ieee80211_sub_if_data *sdata,
                                       const u8 *ie, size_t ie_len,
                                       size_t *offset,
                                       enum nl80211_band band,
                                       u32 rate_mask,
                                       struct cfg80211_chan_def *chandef,
                                       u32 flags)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_supported_band *sband;
        int i, err;
        size_t noffset;
        u32 rate_flags;
        bool have_80mhz = false;

        *offset = 0;

        sband = local->hw.wiphy->bands[band];
        if (WARN_ON_ONCE(!sband))
                return 0;

        rate_flags = ieee80211_chandef_rate_flags(chandef);

        /* For direct scan add S1G IE and consider its override bits */
        if (band == NL80211_BAND_S1GHZ)
                return ieee80211_put_s1g_cap(skb, &sband->s1g_cap);

        err = ieee80211_put_srates_elem(skb, sband, 0, rate_flags,
                                        ~rate_mask, WLAN_EID_SUPP_RATES);
        if (err)
                return err;

        /* insert "request information" if in custom IEs */
        if (ie && ie_len) {
                static const u8 before_extrates[] = {
                        WLAN_EID_SSID,
                        WLAN_EID_SUPP_RATES,
                        WLAN_EID_REQUEST,
                };
                noffset = ieee80211_ie_split(ie, ie_len,
                                             before_extrates,
                                             ARRAY_SIZE(before_extrates),
                                             *offset);
                if (skb_tailroom(skb) < noffset - *offset)
                        return -ENOBUFS;
                skb_put_data(skb, ie + *offset, noffset - *offset);
                *offset = noffset;
        }

        err = ieee80211_put_srates_elem(skb, sband, 0, rate_flags,
                                        ~rate_mask, WLAN_EID_EXT_SUPP_RATES);
        if (err)
                return err;

        if (chandef->chan && sband->band == NL80211_BAND_2GHZ) {
                if (skb_tailroom(skb) < 3)
                        return -ENOBUFS;
                skb_put_u8(skb, WLAN_EID_DS_PARAMS);
                skb_put_u8(skb, 1);
                skb_put_u8(skb,
                           ieee80211_frequency_to_channel(chandef->chan->center_freq));
        }

        if (flags & IEEE80211_PROBE_FLAG_MIN_CONTENT)
                return 0;

        /* insert custom IEs that go before HT */
        if (ie && ie_len) {
                static const u8 before_ht[] = {
                        /*
                         * no need to list the ones split off already
                         * (or generated here)
                         */
                        WLAN_EID_DS_PARAMS,
                        WLAN_EID_SUPPORTED_REGULATORY_CLASSES,
                };
                noffset = ieee80211_ie_split(ie, ie_len,
                                             before_ht, ARRAY_SIZE(before_ht),
                                             *offset);
                if (skb_tailroom(skb) < noffset - *offset)
                        return -ENOBUFS;
                skb_put_data(skb, ie + *offset, noffset - *offset);
                *offset = noffset;
        }

        if (sband->ht_cap.ht_supported) {
                u8 *pos;

                if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_ht_cap))
                        return -ENOBUFS;

                pos = skb_put(skb, 2 + sizeof(struct ieee80211_ht_cap));
                ieee80211_ie_build_ht_cap(pos, &sband->ht_cap,
                                          sband->ht_cap.cap);
        }

        /* insert custom IEs that go before VHT */
        if (ie && ie_len) {
                static const u8 before_vht[] = {
                        /*
                         * no need to list the ones split off already
                         * (or generated here)
                         */
                        WLAN_EID_BSS_COEX_2040,
                        WLAN_EID_EXT_CAPABILITY,
                        WLAN_EID_SSID_LIST,
                        WLAN_EID_CHANNEL_USAGE,
                        WLAN_EID_INTERWORKING,
                        WLAN_EID_MESH_ID,
                        /* 60 GHz (Multi-band, DMG, MMS) can't happen */
                };
                noffset = ieee80211_ie_split(ie, ie_len,
                                             before_vht, ARRAY_SIZE(before_vht),
                                             *offset);
                if (skb_tailroom(skb) < noffset - *offset)
                        return -ENOBUFS;
                skb_put_data(skb, ie + *offset, noffset - *offset);
                *offset = noffset;
        }

        /* Check if any channel in this sband supports at least 80 MHz */
        for (i = 0; i < sband->n_channels; i++) {
                if (sband->channels[i].flags & (IEEE80211_CHAN_DISABLED |
                                                IEEE80211_CHAN_NO_80MHZ))
                        continue;

                have_80mhz = true;
                break;
        }

        if (sband->vht_cap.vht_supported && have_80mhz) {
                u8 *pos;

                if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_vht_cap))
                        return -ENOBUFS;

                pos = skb_put(skb, 2 + sizeof(struct ieee80211_vht_cap));
                ieee80211_ie_build_vht_cap(pos, &sband->vht_cap,
                                           sband->vht_cap.cap);
        }

        /* insert custom IEs that go before HE */
        if (ie && ie_len) {
                static const u8 before_he[] = {
                        /*
                         * no need to list the ones split off before VHT
                         * or generated here
                         */
                        WLAN_EID_EXTENSION, WLAN_EID_EXT_FILS_REQ_PARAMS,
                        WLAN_EID_AP_CSN,
                        /* TODO: add 11ah/11aj/11ak elements */
                };
                noffset = ieee80211_ie_split(ie, ie_len,
                                             before_he, ARRAY_SIZE(before_he),
                                             *offset);
                if (skb_tailroom(skb) < noffset - *offset)
                        return -ENOBUFS;
                skb_put_data(skb, ie + *offset, noffset - *offset);
                *offset = noffset;
        }

        if (cfg80211_any_usable_channels(local->hw.wiphy, BIT(sband->band),
                                         IEEE80211_CHAN_NO_HE)) {
                err = ieee80211_put_he_cap(skb, sdata, sband, NULL);
                if (err)
                        return err;
        }

        if (cfg80211_any_usable_channels(local->hw.wiphy, BIT(sband->band),
                                         IEEE80211_CHAN_NO_HE |
                                         IEEE80211_CHAN_NO_EHT)) {
                err = ieee80211_put_eht_cap(skb, sdata, sband, NULL);
                if (err)
                        return err;
        }

        err = ieee80211_put_he_6ghz_cap(skb, sdata, IEEE80211_SMPS_OFF);
        if (err)
                return err;

        /*
         * If adding more here, adjust code in main.c
         * that calculates local->scan_ies_len.
         */

        return 0;
}

static int ieee80211_put_preq_ies(struct sk_buff *skb,
                                  struct ieee80211_sub_if_data *sdata,
                                  struct ieee80211_scan_ies *ie_desc,
                                  const u8 *ie, size_t ie_len,
                                  u8 bands_used, u32 *rate_masks,
                                  struct cfg80211_chan_def *chandef,
                                  u32 flags)
{
        size_t custom_ie_offset = 0;
        int i, err;

        memset(ie_desc, 0, sizeof(*ie_desc));

        for (i = 0; i < NUM_NL80211_BANDS; i++) {
                if (bands_used & BIT(i)) {
                        ie_desc->ies[i] = skb_tail_pointer(skb);
                        err = ieee80211_put_preq_ies_band(skb, sdata,
                                                          ie, ie_len,
                                                          &custom_ie_offset,
                                                          i, rate_masks[i],
                                                          chandef, flags);
                        if (err)
                                return err;
                        ie_desc->len[i] = skb_tail_pointer(skb) -
                                          ie_desc->ies[i];
                }
        }

        /* add any remaining custom IEs */
        if (ie && ie_len) {
                if (WARN_ONCE(skb_tailroom(skb) < ie_len - custom_ie_offset,
                              "not enough space for preq custom IEs\n"))
                        return -ENOBUFS;
                ie_desc->common_ies = skb_tail_pointer(skb);
                skb_put_data(skb, ie + custom_ie_offset,
                             ie_len - custom_ie_offset);
                ie_desc->common_ie_len = skb_tail_pointer(skb) -
                                         ie_desc->common_ies;
        }

        return 0;
};

int ieee80211_build_preq_ies(struct ieee80211_sub_if_data *sdata, u8 *buffer,
                             size_t buffer_len,
                             struct ieee80211_scan_ies *ie_desc,
                             const u8 *ie, size_t ie_len,
                             u8 bands_used, u32 *rate_masks,
                             struct cfg80211_chan_def *chandef,
                             u32 flags)
{
        struct sk_buff *skb = alloc_skb(buffer_len, GFP_KERNEL);
        uintptr_t offs;
        int ret, i;
        u8 *start;

        if (!skb)
                return -ENOMEM;

        start = skb_tail_pointer(skb);
        memset(start, 0, skb_tailroom(skb));
        ret = ieee80211_put_preq_ies(skb, sdata, ie_desc, ie, ie_len,
                                     bands_used, rate_masks, chandef,
                                     flags);
        if (ret < 0) {
                goto out;
        }

        if (skb->len > buffer_len) {
                ret = -ENOBUFS;
                goto out;
        }

        memcpy(buffer, start, skb->len);

        /* adjust ie_desc for copy */
        for (i = 0; i < NUM_NL80211_BANDS; i++) {
                offs = ie_desc->ies[i] - start;
                ie_desc->ies[i] = buffer + offs;
        }
        offs = ie_desc->common_ies - start;
        ie_desc->common_ies = buffer + offs;

        ret = skb->len;
out:
        consume_skb(skb);
        return ret;
}

struct sk_buff *ieee80211_build_probe_req(struct ieee80211_sub_if_data *sdata,
                                          const u8 *src, const u8 *dst,
                                          u32 ratemask,
                                          struct ieee80211_channel *chan,
                                          const u8 *ssid, size_t ssid_len,
                                          const u8 *ie, size_t ie_len,
                                          u32 flags)
{
        struct ieee80211_local *local = sdata->local;
        struct cfg80211_chan_def chandef;
        struct sk_buff *skb;
        struct ieee80211_mgmt *mgmt;
        u32 rate_masks[NUM_NL80211_BANDS] = {};
        struct ieee80211_scan_ies dummy_ie_desc;

        /*
         * Do not send DS Channel parameter for directed probe requests
         * in order to maximize the chance that we get a response.  Some
         * badly-behaved APs don't respond when this parameter is included.
         */
        chandef.width = sdata->vif.bss_conf.chanreq.oper.width;
        if (flags & IEEE80211_PROBE_FLAG_DIRECTED)
                chandef.chan = NULL;
        else
                chandef.chan = chan;

        skb = ieee80211_probereq_get(&local->hw, src, ssid, ssid_len,
                                     local->scan_ies_len + ie_len);
        if (!skb)
                return NULL;

        rate_masks[chan->band] = ratemask;
        ieee80211_put_preq_ies(skb, sdata, &dummy_ie_desc,
                               ie, ie_len, BIT(chan->band),
                               rate_masks, &chandef, flags);

        if (dst) {
                mgmt = (struct ieee80211_mgmt *) skb->data;
                memcpy(mgmt->da, dst, ETH_ALEN);
                memcpy(mgmt->bssid, dst, ETH_ALEN);
        }

        IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT;

        return skb;
}

u32 ieee80211_sta_get_rates(struct ieee80211_sub_if_data *sdata,
                            struct ieee802_11_elems *elems,
                            enum nl80211_band band, u32 *basic_rates)
{
        struct ieee80211_supported_band *sband;
        size_t num_rates;
        u32 supp_rates, rate_flags;
        int i, j;

        sband = sdata->local->hw.wiphy->bands[band];
        if (WARN_ON(!sband))
                return 1;

        rate_flags =
                ieee80211_chandef_rate_flags(&sdata->vif.bss_conf.chanreq.oper);

        num_rates = sband->n_bitrates;
        supp_rates = 0;
        for (i = 0; i < elems->supp_rates_len +
                     elems->ext_supp_rates_len; i++) {
                u8 rate = 0;
                int own_rate;
                bool is_basic;
                if (i < elems->supp_rates_len)
                        rate = elems->supp_rates[i];
                else if (elems->ext_supp_rates)
                        rate = elems->ext_supp_rates
                                [i - elems->supp_rates_len];
                own_rate = 5 * (rate & 0x7f);
                is_basic = !!(rate & 0x80);

                if (is_basic && (rate & 0x7f) == BSS_MEMBERSHIP_SELECTOR_HT_PHY)
                        continue;

                for (j = 0; j < num_rates; j++) {
                        int brate;
                        if ((rate_flags & sband->bitrates[j].flags)
                            != rate_flags)
                                continue;

                        brate = sband->bitrates[j].bitrate;

                        if (brate == own_rate) {
                                supp_rates |= BIT(j);
                                if (basic_rates && is_basic)
                                        *basic_rates |= BIT(j);
                        }
                }
        }
        return supp_rates;
}

void ieee80211_stop_device(struct ieee80211_local *local, bool suspend)
{
        local_bh_disable();
        ieee80211_handle_queued_frames(local);
        local_bh_enable();

        ieee80211_led_radio(local, false);
        ieee80211_mod_tpt_led_trig(local, 0, IEEE80211_TPT_LEDTRIG_FL_RADIO);

        wiphy_work_cancel(local->hw.wiphy, &local->reconfig_filter);

        flush_workqueue(local->workqueue);
        wiphy_work_flush(local->hw.wiphy, NULL);
        drv_stop(local, suspend);
}

static void ieee80211_flush_completed_scan(struct ieee80211_local *local,
                                           bool aborted)
{
        /* It's possible that we don't handle the scan completion in
         * time during suspend, so if it's still marked as completed
         * here, queue the work and flush it to clean things up.
         * Instead of calling the worker function directly here, we
         * really queue it to avoid potential races with other flows
         * scheduling the same work.
         */
        if (test_bit(SCAN_COMPLETED, &local->scanning)) {
                /* If coming from reconfiguration failure, abort the scan so
                 * we don't attempt to continue a partial HW scan - which is
                 * possible otherwise if (e.g.) the 2.4 GHz portion was the
                 * completed scan, and a 5 GHz portion is still pending.
                 */
                if (aborted)
                        set_bit(SCAN_ABORTED, &local->scanning);
                wiphy_delayed_work_queue(local->hw.wiphy, &local->scan_work, 0);
                wiphy_delayed_work_flush(local->hw.wiphy, &local->scan_work);
        }
}

static void ieee80211_handle_reconfig_failure(struct ieee80211_local *local)
{
        struct ieee80211_sub_if_data *sdata;
        struct ieee80211_chanctx *ctx;

        lockdep_assert_wiphy(local->hw.wiphy);

        /*
         * We get here if during resume the device can't be restarted properly.
         * We might also get here if this happens during HW reset, which is a
         * slightly different situation and we need to drop all connections in
         * the latter case.
         *
         * Ask cfg80211 to turn off all interfaces, this will result in more
         * warnings but at least we'll then get into a clean stopped state.
         */

        local->resuming = false;
        local->suspended = false;
        local->in_reconfig = false;
        local->reconfig_failure = true;

        ieee80211_flush_completed_scan(local, true);

        /* scheduled scan clearly can't be running any more, but tell
         * cfg80211 and clear local state
         */
        ieee80211_sched_scan_end(local);

        list_for_each_entry(sdata, &local->interfaces, list)
                sdata->flags &= ~IEEE80211_SDATA_IN_DRIVER;

        /* Mark channel contexts as not being in the driver any more to avoid
         * removing them from the driver during the shutdown process...
         */
        list_for_each_entry(ctx, &local->chanctx_list, list)
                ctx->driver_present = false;
}

static void ieee80211_assign_chanctx(struct ieee80211_local *local,
                                     struct ieee80211_sub_if_data *sdata,
                                     struct ieee80211_link_data *link)
{
        struct ieee80211_chanctx_conf *conf;
        struct ieee80211_chanctx *ctx;

        lockdep_assert_wiphy(local->hw.wiphy);

        conf = rcu_dereference_protected(link->conf->chanctx_conf,
                                         lockdep_is_held(&local->hw.wiphy->mtx));
        if (conf) {
                ctx = container_of(conf, struct ieee80211_chanctx, conf);
                drv_assign_vif_chanctx(local, sdata, link->conf, ctx);
        }
}

static void ieee80211_reconfig_stations(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct sta_info *sta;

        lockdep_assert_wiphy(local->hw.wiphy);

        /* add STAs back */
        list_for_each_entry(sta, &local->sta_list, list) {
                enum ieee80211_sta_state state;

                if (!sta->uploaded || sta->sdata != sdata)
                        continue;

                for (state = IEEE80211_STA_NOTEXIST;
                     state < sta->sta_state; state++)
                        WARN_ON(drv_sta_state(local, sta->sdata, sta, state,
                                              state + 1));
        }
}

static int ieee80211_reconfig_nan(struct ieee80211_sub_if_data *sdata)
{
        struct cfg80211_nan_func *func, **funcs;
        int res, id, i = 0;

        res = drv_start_nan(sdata->local, sdata,
                            &sdata->u.nan.conf);
        if (WARN_ON(res))
                return res;

        funcs = kcalloc(sdata->local->hw.max_nan_de_entries + 1,
                        sizeof(*funcs),
                        GFP_KERNEL);
        if (!funcs)
                return -ENOMEM;

        /* Add all the functions:
         * This is a little bit ugly. We need to call a potentially sleeping
         * callback for each NAN function, so we can't hold the spinlock.
         */
        spin_lock_bh(&sdata->u.nan.func_lock);

        idr_for_each_entry(&sdata->u.nan.function_inst_ids, func, id)
                funcs[i++] = func;

        spin_unlock_bh(&sdata->u.nan.func_lock);

        for (i = 0; funcs[i]; i++) {
                res = drv_add_nan_func(sdata->local, sdata, funcs[i]);
                if (WARN_ON(res))
                        ieee80211_nan_func_terminated(&sdata->vif,
                                                      funcs[i]->instance_id,
                                                      NL80211_NAN_FUNC_TERM_REASON_ERROR,
                                                      GFP_KERNEL);
        }

        kfree(funcs);

        return 0;
}

static void ieee80211_reconfig_ap_links(struct ieee80211_local *local,
                                        struct ieee80211_sub_if_data *sdata,
                                        u64 changed)
{
        int link_id;

        for (link_id = 0; link_id < ARRAY_SIZE(sdata->link); link_id++) {
                struct ieee80211_link_data *link;

                if (!(sdata->vif.active_links & BIT(link_id)))
                        continue;

                link = sdata_dereference(sdata->link[link_id], sdata);
                if (!link)
                        continue;

                if (rcu_access_pointer(link->u.ap.beacon))
                        drv_start_ap(local, sdata, link->conf);

                if (!link->conf->enable_beacon)
                        continue;

                changed |= BSS_CHANGED_BEACON |
                           BSS_CHANGED_BEACON_ENABLED;

                ieee80211_link_info_change_notify(sdata, link, changed);
        }
}

int ieee80211_reconfig(struct ieee80211_local *local)
{
        struct ieee80211_hw *hw = &local->hw;
        struct ieee80211_sub_if_data *sdata;
        struct ieee80211_chanctx *ctx;
        struct sta_info *sta;
        int res, i;
        bool reconfig_due_to_wowlan = false;
        struct ieee80211_sub_if_data *sched_scan_sdata;
        struct cfg80211_sched_scan_request *sched_scan_req;
        bool sched_scan_stopped = false;
        bool suspended = local->suspended;
        bool in_reconfig = false;

        lockdep_assert_wiphy(local->hw.wiphy);

        /* nothing to do if HW shouldn't run */
        if (!local->open_count)
                goto wake_up;

#ifdef CONFIG_PM
        if (suspended)
                local->resuming = true;

        if (local->wowlan) {
                /*
                 * In the wowlan case, both mac80211 and the device
                 * are functional when the resume op is called, so
                 * clear local->suspended so the device could operate
                 * normally (e.g. pass rx frames).
                 */
                local->suspended = false;
                res = drv_resume(local);
                local->wowlan = false;
                if (res < 0) {
                        local->resuming = false;
                        return res;
                }
                if (res == 0)
                        goto wake_up;
                WARN_ON(res > 1);
                /*
                 * res is 1, which means the driver requested
                 * to go through a regular reset on wakeup.
                 * restore local->suspended in this case.
                 */
                reconfig_due_to_wowlan = true;
                local->suspended = true;
        }
#endif

        /*
         * In case of hw_restart during suspend (without wowlan),
         * cancel restart work, as we are reconfiguring the device
         * anyway.
         * Note that restart_work is scheduled on a frozen workqueue,
         * so we can't deadlock in this case.
         */
        if (suspended && local->in_reconfig && !reconfig_due_to_wowlan)
                cancel_work_sync(&local->restart_work);

        local->started = false;

        /*
         * Upon resume hardware can sometimes be goofy due to
         * various platform / driver / bus issues, so restarting
         * the device may at times not work immediately. Propagate
         * the error.
         */
        res = drv_start(local);
        if (res) {
                if (suspended)
                        WARN(1, "Hardware became unavailable upon resume. This could be a software issue prior to suspend or a hardware issue.\n");
                else
                        WARN(1, "Hardware became unavailable during restart.\n");
                ieee80211_handle_reconfig_failure(local);
                return res;
        }

        /* setup fragmentation threshold */
        drv_set_frag_threshold(local, hw->wiphy->frag_threshold);

        /* setup RTS threshold */
        drv_set_rts_threshold(local, hw->wiphy->rts_threshold);

        /* reset coverage class */
        drv_set_coverage_class(local, hw->wiphy->coverage_class);

        ieee80211_led_radio(local, true);
        ieee80211_mod_tpt_led_trig(local,
                                   IEEE80211_TPT_LEDTRIG_FL_RADIO, 0);

        /* add interfaces */
        sdata = wiphy_dereference(local->hw.wiphy, local->monitor_sdata);
        if (sdata && ieee80211_hw_check(&local->hw, WANT_MONITOR_VIF)) {
                /* in HW restart it exists already */
                WARN_ON(local->resuming);
                res = drv_add_interface(local, sdata);
                if (WARN_ON(res)) {
                        RCU_INIT_POINTER(local->monitor_sdata, NULL);
                        synchronize_net();
                        kfree(sdata);
                }
        }

        list_for_each_entry(sdata, &local->interfaces, list) {
                if (sdata->vif.type == NL80211_IFTYPE_MONITOR &&
                    !ieee80211_hw_check(&local->hw, NO_VIRTUAL_MONITOR))
                        continue;
                if (sdata->vif.type != NL80211_IFTYPE_AP_VLAN &&
                    ieee80211_sdata_running(sdata)) {
                        res = drv_add_interface(local, sdata);
                        if (WARN_ON(res))
                                break;
                }
        }

        /* If adding any of the interfaces failed above, roll back and
         * report failure.
         */
        if (res) {
                list_for_each_entry_continue_reverse(sdata, &local->interfaces,
                                                     list) {
                        if (sdata->vif.type == NL80211_IFTYPE_MONITOR &&
                            !ieee80211_hw_check(&local->hw, NO_VIRTUAL_MONITOR))
                                continue;
                        if (sdata->vif.type != NL80211_IFTYPE_AP_VLAN &&
                            ieee80211_sdata_running(sdata))
                                drv_remove_interface(local, sdata);
                }
                ieee80211_handle_reconfig_failure(local);
                return res;
        }

        /* add channel contexts */
        list_for_each_entry(ctx, &local->chanctx_list, list)
                if (ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER)
                        WARN_ON(drv_add_chanctx(local, ctx));

        sdata = wiphy_dereference(local->hw.wiphy, local->monitor_sdata);
        if (sdata && ieee80211_sdata_running(sdata))
                ieee80211_assign_chanctx(local, sdata, &sdata->deflink);

        /* reconfigure hardware */
        ieee80211_hw_config(local, IEEE80211_CONF_CHANGE_LISTEN_INTERVAL |
                                   IEEE80211_CONF_CHANGE_MONITOR |
                                   IEEE80211_CONF_CHANGE_PS |
                                   IEEE80211_CONF_CHANGE_RETRY_LIMITS |
                                   IEEE80211_CONF_CHANGE_IDLE);

        ieee80211_configure_filter(local);

        /* Finally also reconfigure all the BSS information */
        list_for_each_entry(sdata, &local->interfaces, list) {
                /* common change flags for all interface types - link only */
                u64 changed = BSS_CHANGED_ERP_CTS_PROT |
                              BSS_CHANGED_ERP_PREAMBLE |
                              BSS_CHANGED_ERP_SLOT |
                              BSS_CHANGED_HT |
                              BSS_CHANGED_BASIC_RATES |
                              BSS_CHANGED_BEACON_INT |
                              BSS_CHANGED_BSSID |
                              BSS_CHANGED_CQM |
                              BSS_CHANGED_QOS |
                              BSS_CHANGED_TXPOWER |
                              BSS_CHANGED_MCAST_RATE;
                struct ieee80211_link_data *link = NULL;
                unsigned int link_id;
                u32 active_links = 0;

                if (!ieee80211_sdata_running(sdata))
                        continue;

                if (ieee80211_vif_is_mld(&sdata->vif)) {
                        struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS] = {
                                [0] = &sdata->vif.bss_conf,
                        };

                        if (sdata->vif.type == NL80211_IFTYPE_STATION) {
                                /* start with a single active link */
                                active_links = sdata->vif.active_links;
                                link_id = ffs(active_links) - 1;
                                sdata->vif.active_links = BIT(link_id);
                        }

                        drv_change_vif_links(local, sdata, 0,
                                             sdata->vif.active_links,
                                             old);
                }

                sdata->restart_active_links = active_links;

                for (link_id = 0;
                     link_id < ARRAY_SIZE(sdata->vif.link_conf);
                     link_id++) {
                        if (!ieee80211_vif_link_active(&sdata->vif, link_id))
                                continue;

                        link = sdata_dereference(sdata->link[link_id], sdata);
                        if (!link)
                                continue;

                        ieee80211_assign_chanctx(local, sdata, link);
                }

                switch (sdata->vif.type) {
                case NL80211_IFTYPE_AP_VLAN:
                case NL80211_IFTYPE_MONITOR:
                        break;
                case NL80211_IFTYPE_ADHOC:
                        if (sdata->vif.cfg.ibss_joined)
                                WARN_ON(drv_join_ibss(local, sdata));
                        fallthrough;
                default:
                        ieee80211_reconfig_stations(sdata);
                        fallthrough;
                case NL80211_IFTYPE_AP: /* AP stations are handled later */
                        for (i = 0; i < IEEE80211_NUM_ACS; i++)
                                drv_conf_tx(local, &sdata->deflink, i,
                                            &sdata->deflink.tx_conf[i]);
                        break;
                }

                if (sdata->vif.bss_conf.mu_mimo_owner)
                        changed |= BSS_CHANGED_MU_GROUPS;

                if (!ieee80211_vif_is_mld(&sdata->vif))
                        changed |= BSS_CHANGED_IDLE;

                switch (sdata->vif.type) {
                case NL80211_IFTYPE_STATION:
                        if (!ieee80211_vif_is_mld(&sdata->vif)) {
                                changed |= BSS_CHANGED_ASSOC |
                                           BSS_CHANGED_ARP_FILTER |
                                           BSS_CHANGED_PS;

                                /* Re-send beacon info report to the driver */
                                if (sdata->deflink.u.mgd.have_beacon)
                                        changed |= BSS_CHANGED_BEACON_INFO;

                                if (sdata->vif.bss_conf.max_idle_period ||
                                    sdata->vif.bss_conf.protected_keep_alive)
                                        changed |= BSS_CHANGED_KEEP_ALIVE;

                                ieee80211_bss_info_change_notify(sdata,
                                                                 changed);
                        } else if (!WARN_ON(!link)) {
                                ieee80211_link_info_change_notify(sdata, link,
                                                                  changed);
                                changed = BSS_CHANGED_ASSOC |
                                          BSS_CHANGED_IDLE |
                                          BSS_CHANGED_PS |
                                          BSS_CHANGED_ARP_FILTER;
                                ieee80211_vif_cfg_change_notify(sdata, changed);
                        }
                        break;
                case NL80211_IFTYPE_OCB:
                        changed |= BSS_CHANGED_OCB;
                        ieee80211_bss_info_change_notify(sdata, changed);
                        break;
                case NL80211_IFTYPE_ADHOC:
                        changed |= BSS_CHANGED_IBSS;
                        fallthrough;
                case NL80211_IFTYPE_AP:
                        changed |= BSS_CHANGED_P2P_PS;

                        if (ieee80211_vif_is_mld(&sdata->vif))
                                ieee80211_vif_cfg_change_notify(sdata,
                                                                BSS_CHANGED_SSID);
                        else
                                changed |= BSS_CHANGED_SSID;

                        if (sdata->vif.bss_conf.ftm_responder == 1 &&
                            wiphy_ext_feature_isset(sdata->local->hw.wiphy,
                                        NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER))
                                changed |= BSS_CHANGED_FTM_RESPONDER;

                        if (sdata->vif.type == NL80211_IFTYPE_AP) {
                                changed |= BSS_CHANGED_AP_PROBE_RESP;

                                if (ieee80211_vif_is_mld(&sdata->vif)) {
                                        ieee80211_reconfig_ap_links(local,
                                                                    sdata,
                                                                    changed);
                                        break;
                                }

                                if (rcu_access_pointer(sdata->deflink.u.ap.beacon))
                                        drv_start_ap(local, sdata,
                                                     sdata->deflink.conf);
                        }
                        fallthrough;
                case NL80211_IFTYPE_MESH_POINT:
                        if (sdata->vif.bss_conf.enable_beacon) {
                                changed |= BSS_CHANGED_BEACON |
                                           BSS_CHANGED_BEACON_ENABLED;
                                ieee80211_bss_info_change_notify(sdata, changed);
                        }
                        break;
                case NL80211_IFTYPE_NAN:
                        res = ieee80211_reconfig_nan(sdata);
                        if (res < 0) {
                                ieee80211_handle_reconfig_failure(local);
                                return res;
                        }
                        break;
                case NL80211_IFTYPE_AP_VLAN:
                case NL80211_IFTYPE_MONITOR:
                case NL80211_IFTYPE_P2P_DEVICE:
                        /* nothing to do */
                        break;
                case NL80211_IFTYPE_UNSPECIFIED:
                case NUM_NL80211_IFTYPES:
                case NL80211_IFTYPE_P2P_CLIENT:
                case NL80211_IFTYPE_P2P_GO:
                case NL80211_IFTYPE_WDS:
                        WARN_ON(1);
                        break;
                }
        }

        ieee80211_recalc_ps(local);

        /*
         * The sta might be in psm against the ap (e.g. because
         * this was the state before a hw restart), so we
         * explicitly send a null packet in order to make sure
         * it'll sync against the ap (and get out of psm).
         */
        if (!(local->hw.conf.flags & IEEE80211_CONF_PS)) {
                list_for_each_entry(sdata, &local->interfaces, list) {
                        if (sdata->vif.type != NL80211_IFTYPE_STATION)
                                continue;
                        if (!sdata->u.mgd.associated)
                                continue;

                        ieee80211_send_nullfunc(local, sdata, false);
                }
        }

        /* APs are now beaconing, add back stations */
        list_for_each_entry(sdata, &local->interfaces, list) {
                if (!ieee80211_sdata_running(sdata))
                        continue;

                switch (sdata->vif.type) {
                case NL80211_IFTYPE_AP_VLAN:
                case NL80211_IFTYPE_AP:
                        ieee80211_reconfig_stations(sdata);
                        break;
                default:
                        break;
                }
        }

        /* add back keys */
        list_for_each_entry(sdata, &local->interfaces, list)
                ieee80211_reenable_keys(sdata);

        /* re-enable multi-link for client interfaces */
        list_for_each_entry(sdata, &local->interfaces, list) {
                if (sdata->restart_active_links)
                        ieee80211_set_active_links(&sdata->vif,
                                                   sdata->restart_active_links);
                /*
                 * If a link switch was scheduled before the restart, and ran
                 * before reconfig, it will do nothing, so re-schedule.
                 */
                if (sdata->desired_active_links)
                        wiphy_work_queue(sdata->local->hw.wiphy,
                                         &sdata->activate_links_work);
        }

        /* Reconfigure sched scan if it was interrupted by FW restart */
        sched_scan_sdata = rcu_dereference_protected(local->sched_scan_sdata,
                                                lockdep_is_held(&local->hw.wiphy->mtx));
        sched_scan_req = rcu_dereference_protected(local->sched_scan_req,
                                                lockdep_is_held(&local->hw.wiphy->mtx));
        if (sched_scan_sdata && sched_scan_req)
                /*
                 * Sched scan stopped, but we don't want to report it. Instead,
                 * we're trying to reschedule. However, if more than one scan
                 * plan was set, we cannot reschedule since we don't know which
                 * scan plan was currently running (and some scan plans may have
                 * already finished).
                 */
                if (sched_scan_req->n_scan_plans > 1 ||
                    __ieee80211_request_sched_scan_start(sched_scan_sdata,
                                                         sched_scan_req)) {
                        RCU_INIT_POINTER(local->sched_scan_sdata, NULL);
                        RCU_INIT_POINTER(local->sched_scan_req, NULL);
                        sched_scan_stopped = true;
                }

        if (sched_scan_stopped)
                cfg80211_sched_scan_stopped_locked(local->hw.wiphy, 0);

 wake_up:

        if (local->monitors == local->open_count && local->monitors > 0)
                ieee80211_add_virtual_monitor(local);

        /*
         * Clear the WLAN_STA_BLOCK_BA flag so new aggregation
         * sessions can be established after a resume.
         *
         * Also tear down aggregation sessions since reconfiguring
         * them in a hardware restart scenario is not easily done
         * right now, and the hardware will have lost information
         * about the sessions, but we and the AP still think they
         * are active. This is really a workaround though.
         */
        if (ieee80211_hw_check(hw, AMPDU_AGGREGATION)) {
                list_for_each_entry(sta, &local->sta_list, list) {
                        if (!local->resuming)
                                ieee80211_sta_tear_down_BA_sessions(
                                                sta, AGG_STOP_LOCAL_REQUEST);
                        clear_sta_flag(sta, WLAN_STA_BLOCK_BA);
                }
        }

        /*
         * If this is for hw restart things are still running.
         * We may want to change that later, however.
         */
        if (local->open_count && (!suspended || reconfig_due_to_wowlan))
                drv_reconfig_complete(local, IEEE80211_RECONFIG_TYPE_RESTART);

        if (local->in_reconfig) {
                in_reconfig = local->in_reconfig;
                local->in_reconfig = false;
                barrier();

                ieee80211_reconfig_roc(local);

                /* Requeue all works */
                list_for_each_entry(sdata, &local->interfaces, list)
                        wiphy_work_queue(local->hw.wiphy, &sdata->work);
        }

        ieee80211_wake_queues_by_reason(hw, IEEE80211_MAX_QUEUE_MAP,
                                        IEEE80211_QUEUE_STOP_REASON_SUSPEND,
                                        false);

        if (in_reconfig) {
                list_for_each_entry(sdata, &local->interfaces, list) {
                        if (!ieee80211_sdata_running(sdata))
                                continue;
                        if (sdata->vif.type == NL80211_IFTYPE_STATION)
                                ieee80211_sta_restart(sdata);
                }
        }

        if (!suspended)
                return 0;

#ifdef CONFIG_PM
        /* first set suspended false, then resuming */
        local->suspended = false;
        mb();
        local->resuming = false;

        ieee80211_flush_completed_scan(local, false);

        if (local->open_count && !reconfig_due_to_wowlan)
                drv_reconfig_complete(local, IEEE80211_RECONFIG_TYPE_SUSPEND);

        list_for_each_entry(sdata, &local->interfaces, list) {
                if (!ieee80211_sdata_running(sdata))
                        continue;
                if (sdata->vif.type == NL80211_IFTYPE_STATION)
                        ieee80211_sta_restart(sdata);
        }

        mod_timer(&local->sta_cleanup, jiffies + 1);
#else
        WARN_ON(1);
#endif

        return 0;
}

static void ieee80211_reconfig_disconnect(struct ieee80211_vif *vif, u8 flag)
{
        struct ieee80211_sub_if_data *sdata;
        struct ieee80211_local *local;
        struct ieee80211_key *key;

        if (WARN_ON(!vif))
                return;

        sdata = vif_to_sdata(vif);
        local = sdata->local;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (WARN_ON(flag & IEEE80211_SDATA_DISCONNECT_RESUME &&
                    !local->resuming))
                return;

        if (WARN_ON(flag & IEEE80211_SDATA_DISCONNECT_HW_RESTART &&
                    !local->in_reconfig))
                return;

        if (WARN_ON(vif->type != NL80211_IFTYPE_STATION))
                return;

        sdata->flags |= flag;

        list_for_each_entry(key, &sdata->key_list, list)
                key->flags |= KEY_FLAG_TAINTED;
}

void ieee80211_hw_restart_disconnect(struct ieee80211_vif *vif)
{
        ieee80211_reconfig_disconnect(vif, IEEE80211_SDATA_DISCONNECT_HW_RESTART);
}
EXPORT_SYMBOL_GPL(ieee80211_hw_restart_disconnect);

void ieee80211_resume_disconnect(struct ieee80211_vif *vif)
{
        ieee80211_reconfig_disconnect(vif, IEEE80211_SDATA_DISCONNECT_RESUME);
}
EXPORT_SYMBOL_GPL(ieee80211_resume_disconnect);

void ieee80211_recalc_smps(struct ieee80211_sub_if_data *sdata,
                           struct ieee80211_link_data *link)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *chanctx_conf;
        struct ieee80211_chanctx *chanctx;

        lockdep_assert_wiphy(local->hw.wiphy);

        chanctx_conf = rcu_dereference_protected(link->conf->chanctx_conf,
                                                 lockdep_is_held(&local->hw.wiphy->mtx));

        /*
         * This function can be called from a work, thus it may be possible
         * that the chanctx_conf is removed (due to a disconnection, for
         * example).
         * So nothing should be done in such case.
         */
        if (!chanctx_conf)
                return;

        chanctx = container_of(chanctx_conf, struct ieee80211_chanctx, conf);
        ieee80211_recalc_smps_chanctx(local, chanctx);
}

void ieee80211_recalc_min_chandef(struct ieee80211_sub_if_data *sdata,
                                  int link_id)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *chanctx_conf;
        struct ieee80211_chanctx *chanctx;
        int i;

        lockdep_assert_wiphy(local->hw.wiphy);

        for (i = 0; i < ARRAY_SIZE(sdata->vif.link_conf); i++) {
                struct ieee80211_bss_conf *bss_conf;

                if (link_id >= 0 && link_id != i)
                        continue;

                rcu_read_lock();
                bss_conf = rcu_dereference(sdata->vif.link_conf[i]);
                if (!bss_conf) {
                        rcu_read_unlock();
                        continue;
                }

                chanctx_conf = rcu_dereference_protected(bss_conf->chanctx_conf,
                                                         lockdep_is_held(&local->hw.wiphy->mtx));
                /*
                 * Since we hold the wiphy mutex (checked above)
                 * we can take the chanctx_conf pointer out of the
                 * RCU critical section, it cannot go away without
                 * the mutex. Just the way we reached it could - in
                 * theory - go away, but we don't really care and
                 * it really shouldn't happen anyway.
                 */
                rcu_read_unlock();

                if (!chanctx_conf)
                        return;

                chanctx = container_of(chanctx_conf, struct ieee80211_chanctx,
                                       conf);
                ieee80211_recalc_chanctx_min_def(local, chanctx, NULL, false);
        }
}

size_t ieee80211_ie_split_vendor(const u8 *ies, size_t ielen, size_t offset)
{
        size_t pos = offset;

        while (pos < ielen && ies[pos] != WLAN_EID_VENDOR_SPECIFIC)
                pos += 2 + ies[pos + 1];

        return pos;
}

u8 *ieee80211_ie_build_ht_cap(u8 *pos, struct ieee80211_sta_ht_cap *ht_cap,
                              u16 cap)
{
        __le16 tmp;

        *pos++ = WLAN_EID_HT_CAPABILITY;
        *pos++ = sizeof(struct ieee80211_ht_cap);
        memset(pos, 0, sizeof(struct ieee80211_ht_cap));

        /* capability flags */
        tmp = cpu_to_le16(cap);
        memcpy(pos, &tmp, sizeof(u16));
        pos += sizeof(u16);

        /* AMPDU parameters */
        *pos++ = ht_cap->ampdu_factor |
                 (ht_cap->ampdu_density <<
                        IEEE80211_HT_AMPDU_PARM_DENSITY_SHIFT);

        /* MCS set */
        memcpy(pos, &ht_cap->mcs, sizeof(ht_cap->mcs));
        pos += sizeof(ht_cap->mcs);

        /* extended capabilities */
        pos += sizeof(__le16);

        /* BF capabilities */
        pos += sizeof(__le32);

        /* antenna selection */
        pos += sizeof(u8);

        return pos;
}

u8 *ieee80211_ie_build_vht_cap(u8 *pos, struct ieee80211_sta_vht_cap *vht_cap,
                               u32 cap)
{
        __le32 tmp;

        *pos++ = WLAN_EID_VHT_CAPABILITY;
        *pos++ = sizeof(struct ieee80211_vht_cap);
        memset(pos, 0, sizeof(struct ieee80211_vht_cap));

        /* capability flags */
        tmp = cpu_to_le32(cap);
        memcpy(pos, &tmp, sizeof(u32));
        pos += sizeof(u32);

        /* VHT MCS set */
        memcpy(pos, &vht_cap->vht_mcs, sizeof(vht_cap->vht_mcs));
        pos += sizeof(vht_cap->vht_mcs);

        return pos;
}

/* this may return more than ieee80211_put_he_6ghz_cap() will need */
u8 ieee80211_ie_len_he_cap(struct ieee80211_sub_if_data *sdata)
{
        const struct ieee80211_sta_he_cap *he_cap;
        struct ieee80211_supported_band *sband;
        u8 n;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return 0;

        he_cap = ieee80211_get_he_iftype_cap_vif(sband, &sdata->vif);
        if (!he_cap)
                return 0;

        n = ieee80211_he_mcs_nss_size(&he_cap->he_cap_elem);
        return 2 + 1 +
               sizeof(he_cap->he_cap_elem) + n +
               ieee80211_he_ppe_size(he_cap->ppe_thres[0],
                                     he_cap->he_cap_elem.phy_cap_info);
}

static void
ieee80211_get_adjusted_he_cap(const struct ieee80211_conn_settings *conn,
                              const struct ieee80211_sta_he_cap *he_cap,
                              struct ieee80211_he_cap_elem *elem)
{
        u8 ru_limit, max_ru;

        *elem = he_cap->he_cap_elem;

        switch (conn->bw_limit) {
        case IEEE80211_CONN_BW_LIMIT_20:
                ru_limit = IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_242;
                break;
        case IEEE80211_CONN_BW_LIMIT_40:
                ru_limit = IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_484;
                break;
        case IEEE80211_CONN_BW_LIMIT_80:
                ru_limit = IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_996;
                break;
        default:
                ru_limit = IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_2x996;
                break;
        }

        max_ru = elem->phy_cap_info[8] & IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_MASK;
        max_ru = min(max_ru, ru_limit);
        elem->phy_cap_info[8] &= ~IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_MASK;
        elem->phy_cap_info[8] |= max_ru;

        if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_40) {
                elem->phy_cap_info[0] &=
                        ~(IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
                          IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G);
                elem->phy_cap_info[9] &=
                        ~IEEE80211_HE_PHY_CAP9_LONGER_THAN_16_SIGB_OFDM_SYM;
        }

        if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_160) {
                elem->phy_cap_info[0] &=
                        ~(IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
                          IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G);
                elem->phy_cap_info[5] &=
                        ~IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_ABOVE_80MHZ_MASK;
                elem->phy_cap_info[7] &=
                        ~(IEEE80211_HE_PHY_CAP7_STBC_TX_ABOVE_80MHZ |
                          IEEE80211_HE_PHY_CAP7_STBC_RX_ABOVE_80MHZ);
        }
}

int ieee80211_put_he_cap(struct sk_buff *skb,
                         struct ieee80211_sub_if_data *sdata,
                         const struct ieee80211_supported_band *sband,
                         const struct ieee80211_conn_settings *conn)
{
        const struct ieee80211_sta_he_cap *he_cap;
        struct ieee80211_he_cap_elem elem;
        u8 *len;
        u8 n;
        u8 ie_len;

        if (!conn)
                conn = &ieee80211_conn_settings_unlimited;

        he_cap = ieee80211_get_he_iftype_cap_vif(sband, &sdata->vif);
        if (!he_cap)
                return 0;

        /* modify on stack first to calculate 'n' and 'ie_len' correctly */
        ieee80211_get_adjusted_he_cap(conn, he_cap, &elem);

        n = ieee80211_he_mcs_nss_size(&elem);
        ie_len = 2 + 1 +
                 sizeof(he_cap->he_cap_elem) + n +
                 ieee80211_he_ppe_size(he_cap->ppe_thres[0],
                                       he_cap->he_cap_elem.phy_cap_info);

        if (skb_tailroom(skb) < ie_len)
                return -ENOBUFS;

        skb_put_u8(skb, WLAN_EID_EXTENSION);
        len = skb_put(skb, 1); /* We'll set the size later below */
        skb_put_u8(skb, WLAN_EID_EXT_HE_CAPABILITY);

        /* Fixed data */
        skb_put_data(skb, &elem, sizeof(elem));

        skb_put_data(skb, &he_cap->he_mcs_nss_supp, n);

        /* Check if PPE Threshold should be present */
        if ((he_cap->he_cap_elem.phy_cap_info[6] &
             IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT) == 0)
                goto end;

        /*
         * Calculate how many PPET16/PPET8 pairs are to come. Algorithm:
         * (NSS_M1 + 1) x (num of 1 bits in RU_INDEX_BITMASK)
         */
        n = hweight8(he_cap->ppe_thres[0] &
                     IEEE80211_PPE_THRES_RU_INDEX_BITMASK_MASK);
        n *= (1 + ((he_cap->ppe_thres[0] & IEEE80211_PPE_THRES_NSS_MASK) >>
                   IEEE80211_PPE_THRES_NSS_POS));

        /*
         * Each pair is 6 bits, and we need to add the 7 "header" bits to the
         * total size.
         */
        n = (n * IEEE80211_PPE_THRES_INFO_PPET_SIZE * 2) + 7;
        n = DIV_ROUND_UP(n, 8);

        /* Copy PPE Thresholds */
        skb_put_data(skb, &he_cap->ppe_thres, n);

end:
        *len = skb_tail_pointer(skb) - len - 1;
        return 0;
}

int ieee80211_put_he_6ghz_cap(struct sk_buff *skb,
                              struct ieee80211_sub_if_data *sdata,
                              enum ieee80211_smps_mode smps_mode)
{
        struct ieee80211_supported_band *sband;
        const struct ieee80211_sband_iftype_data *iftd;
        enum nl80211_iftype iftype = ieee80211_vif_type_p2p(&sdata->vif);
        __le16 cap;

        if (!cfg80211_any_usable_channels(sdata->local->hw.wiphy,
                                          BIT(NL80211_BAND_6GHZ),
                                          IEEE80211_CHAN_NO_HE))
                return 0;

        sband = sdata->local->hw.wiphy->bands[NL80211_BAND_6GHZ];

        iftd = ieee80211_get_sband_iftype_data(sband, iftype);
        if (!iftd)
                return 0;

        /* Check for device HE 6 GHz capability before adding element */
        if (!iftd->he_6ghz_capa.capa)
                return 0;

        cap = iftd->he_6ghz_capa.capa;
        cap &= cpu_to_le16(~IEEE80211_HE_6GHZ_CAP_SM_PS);

        switch (smps_mode) {
        case IEEE80211_SMPS_AUTOMATIC:
        case IEEE80211_SMPS_NUM_MODES:
                WARN_ON(1);
                fallthrough;
        case IEEE80211_SMPS_OFF:
                cap |= le16_encode_bits(WLAN_HT_CAP_SM_PS_DISABLED,
                                        IEEE80211_HE_6GHZ_CAP_SM_PS);
                break;
        case IEEE80211_SMPS_STATIC:
                cap |= le16_encode_bits(WLAN_HT_CAP_SM_PS_STATIC,
                                        IEEE80211_HE_6GHZ_CAP_SM_PS);
                break;
        case IEEE80211_SMPS_DYNAMIC:
                cap |= le16_encode_bits(WLAN_HT_CAP_SM_PS_DYNAMIC,
                                        IEEE80211_HE_6GHZ_CAP_SM_PS);
                break;
        }

        if (skb_tailroom(skb) < 2 + 1 + sizeof(cap))
                return -ENOBUFS;

        skb_put_u8(skb, WLAN_EID_EXTENSION);
        skb_put_u8(skb, 1 + sizeof(cap));
        skb_put_u8(skb, WLAN_EID_EXT_HE_6GHZ_CAPA);
        skb_put_data(skb, &cap, sizeof(cap));
        return 0;
}

u8 *ieee80211_ie_build_ht_oper(u8 *pos, struct ieee80211_sta_ht_cap *ht_cap,
                               const struct cfg80211_chan_def *chandef,
                               u16 prot_mode, bool rifs_mode)
{
        struct ieee80211_ht_operation *ht_oper;
        /* Build HT Information */
        *pos++ = WLAN_EID_HT_OPERATION;
        *pos++ = sizeof(struct ieee80211_ht_operation);
        ht_oper = (struct ieee80211_ht_operation *)pos;
        ht_oper->primary_chan = ieee80211_frequency_to_channel(
                                        chandef->chan->center_freq);
        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_160:
        case NL80211_CHAN_WIDTH_80P80:
        case NL80211_CHAN_WIDTH_80:
        case NL80211_CHAN_WIDTH_40:
                if (chandef->center_freq1 > chandef->chan->center_freq)
                        ht_oper->ht_param = IEEE80211_HT_PARAM_CHA_SEC_ABOVE;
                else
                        ht_oper->ht_param = IEEE80211_HT_PARAM_CHA_SEC_BELOW;
                break;
        case NL80211_CHAN_WIDTH_320:
                /* HT information element should not be included on 6GHz */
                WARN_ON(1);
                return pos;
        default:
                ht_oper->ht_param = IEEE80211_HT_PARAM_CHA_SEC_NONE;
                break;
        }
        if (ht_cap->cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 &&
            chandef->width != NL80211_CHAN_WIDTH_20_NOHT &&
            chandef->width != NL80211_CHAN_WIDTH_20)
                ht_oper->ht_param |= IEEE80211_HT_PARAM_CHAN_WIDTH_ANY;

        if (rifs_mode)
                ht_oper->ht_param |= IEEE80211_HT_PARAM_RIFS_MODE;

        ht_oper->operation_mode = cpu_to_le16(prot_mode);
        ht_oper->stbc_param = 0x0000;

        /* It seems that Basic MCS set and Supported MCS set
           are identical for the first 10 bytes */
        memset(&ht_oper->basic_set, 0, 16);
        memcpy(&ht_oper->basic_set, &ht_cap->mcs, 10);

        return pos + sizeof(struct ieee80211_ht_operation);
}

void ieee80211_ie_build_wide_bw_cs(u8 *pos,
                                   const struct cfg80211_chan_def *chandef)
{
        *pos++ = WLAN_EID_WIDE_BW_CHANNEL_SWITCH;        /* EID */
        *pos++ = 3;                                        /* IE length */
        /* New channel width */
        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_80:
                *pos++ = IEEE80211_VHT_CHANWIDTH_80MHZ;
                break;
        case NL80211_CHAN_WIDTH_160:
                *pos++ = IEEE80211_VHT_CHANWIDTH_160MHZ;
                break;
        case NL80211_CHAN_WIDTH_80P80:
                *pos++ = IEEE80211_VHT_CHANWIDTH_80P80MHZ;
                break;
        case NL80211_CHAN_WIDTH_320:
                /* The behavior is not defined for 320 MHz channels */
                WARN_ON(1);
                fallthrough;
        default:
                *pos++ = IEEE80211_VHT_CHANWIDTH_USE_HT;
        }

        /* new center frequency segment 0 */
        *pos++ = ieee80211_frequency_to_channel(chandef->center_freq1);
        /* new center frequency segment 1 */
        if (chandef->center_freq2)
                *pos++ = ieee80211_frequency_to_channel(chandef->center_freq2);
        else
                *pos++ = 0;
}

u8 *ieee80211_ie_build_vht_oper(u8 *pos, struct ieee80211_sta_vht_cap *vht_cap,
                                const struct cfg80211_chan_def *chandef)
{
        struct ieee80211_vht_operation *vht_oper;

        *pos++ = WLAN_EID_VHT_OPERATION;
        *pos++ = sizeof(struct ieee80211_vht_operation);
        vht_oper = (struct ieee80211_vht_operation *)pos;
        vht_oper->center_freq_seg0_idx = ieee80211_frequency_to_channel(
                                                        chandef->center_freq1);
        if (chandef->center_freq2)
                vht_oper->center_freq_seg1_idx =
                        ieee80211_frequency_to_channel(chandef->center_freq2);
        else
                vht_oper->center_freq_seg1_idx = 0x00;

        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_160:
                /*
                 * Convert 160 MHz channel width to new style as interop
                 * workaround.
                 */
                vht_oper->chan_width = IEEE80211_VHT_CHANWIDTH_80MHZ;
                vht_oper->center_freq_seg1_idx = vht_oper->center_freq_seg0_idx;
                if (chandef->chan->center_freq < chandef->center_freq1)
                        vht_oper->center_freq_seg0_idx -= 8;
                else
                        vht_oper->center_freq_seg0_idx += 8;
                break;
        case NL80211_CHAN_WIDTH_80P80:
                /*
                 * Convert 80+80 MHz channel width to new style as interop
                 * workaround.
                 */
                vht_oper->chan_width = IEEE80211_VHT_CHANWIDTH_80MHZ;
                break;
        case NL80211_CHAN_WIDTH_80:
                vht_oper->chan_width = IEEE80211_VHT_CHANWIDTH_80MHZ;
                break;
        case NL80211_CHAN_WIDTH_320:
                /* VHT information element should not be included on 6GHz */
                WARN_ON(1);
                return pos;
        default:
                vht_oper->chan_width = IEEE80211_VHT_CHANWIDTH_USE_HT;
                break;
        }

        /* don't require special VHT peer rates */
        vht_oper->basic_mcs_set = cpu_to_le16(0xffff);

        return pos + sizeof(struct ieee80211_vht_operation);
}

u8 *ieee80211_ie_build_he_oper(u8 *pos, const struct cfg80211_chan_def *chandef)
{
        struct ieee80211_he_operation *he_oper;
        struct ieee80211_he_6ghz_oper *he_6ghz_op;
        u32 he_oper_params;
        u8 ie_len = 1 + sizeof(struct ieee80211_he_operation);

        if (chandef->chan->band == NL80211_BAND_6GHZ)
                ie_len += sizeof(struct ieee80211_he_6ghz_oper);

        *pos++ = WLAN_EID_EXTENSION;
        *pos++ = ie_len;
        *pos++ = WLAN_EID_EXT_HE_OPERATION;

        he_oper_params = 0;
        he_oper_params |= u32_encode_bits(1023, /* disabled */
                                IEEE80211_HE_OPERATION_RTS_THRESHOLD_MASK);
        he_oper_params |= u32_encode_bits(1,
                                IEEE80211_HE_OPERATION_ER_SU_DISABLE);
        he_oper_params |= u32_encode_bits(1,
                                IEEE80211_HE_OPERATION_BSS_COLOR_DISABLED);
        if (chandef->chan->band == NL80211_BAND_6GHZ)
                he_oper_params |= u32_encode_bits(1,
                                IEEE80211_HE_OPERATION_6GHZ_OP_INFO);

        he_oper = (struct ieee80211_he_operation *)pos;
        he_oper->he_oper_params = cpu_to_le32(he_oper_params);

        /* don't require special HE peer rates */
        he_oper->he_mcs_nss_set = cpu_to_le16(0xffff);
        pos += sizeof(struct ieee80211_he_operation);

        if (chandef->chan->band != NL80211_BAND_6GHZ)
                goto out;

        /* TODO add VHT operational */
        he_6ghz_op = (struct ieee80211_he_6ghz_oper *)pos;
        he_6ghz_op->minrate = 6; /* 6 Mbps */
        he_6ghz_op->primary =
                ieee80211_frequency_to_channel(chandef->chan->center_freq);
        he_6ghz_op->ccfs0 =
                ieee80211_frequency_to_channel(chandef->center_freq1);
        if (chandef->center_freq2)
                he_6ghz_op->ccfs1 =
                        ieee80211_frequency_to_channel(chandef->center_freq2);
        else
                he_6ghz_op->ccfs1 = 0;

        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_320:
                /*
                 * TODO: mesh operation is not defined over 6GHz 320 MHz
                 * channels.
                 */
                WARN_ON(1);
                break;
        case NL80211_CHAN_WIDTH_160:
                /* Convert 160 MHz channel width to new style as interop
                 * workaround.
                 */
                he_6ghz_op->control =
                        IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_160MHZ;
                he_6ghz_op->ccfs1 = he_6ghz_op->ccfs0;
                if (chandef->chan->center_freq < chandef->center_freq1)
                        he_6ghz_op->ccfs0 -= 8;
                else
                        he_6ghz_op->ccfs0 += 8;
                fallthrough;
        case NL80211_CHAN_WIDTH_80P80:
                he_6ghz_op->control =
                        IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_160MHZ;
                break;
        case NL80211_CHAN_WIDTH_80:
                he_6ghz_op->control =
                        IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_80MHZ;
                break;
        case NL80211_CHAN_WIDTH_40:
                he_6ghz_op->control =
                        IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_40MHZ;
                break;
        default:
                he_6ghz_op->control =
                        IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_20MHZ;
                break;
        }

        pos += sizeof(struct ieee80211_he_6ghz_oper);

out:
        return pos;
}

u8 *ieee80211_ie_build_eht_oper(u8 *pos, const struct cfg80211_chan_def *chandef,
                                const struct ieee80211_sta_eht_cap *eht_cap)

{
        const struct ieee80211_eht_mcs_nss_supp_20mhz_only *eht_mcs_nss =
                                        &eht_cap->eht_mcs_nss_supp.only_20mhz;
        struct ieee80211_eht_operation *eht_oper;
        struct ieee80211_eht_operation_info *eht_oper_info;
        u8 eht_oper_len = offsetof(struct ieee80211_eht_operation, optional);
        u8 eht_oper_info_len =
                offsetof(struct ieee80211_eht_operation_info, optional);
        u8 chan_width = 0;

        *pos++ = WLAN_EID_EXTENSION;
        *pos++ = 1 + eht_oper_len + eht_oper_info_len;
        *pos++ = WLAN_EID_EXT_EHT_OPERATION;

        eht_oper = (struct ieee80211_eht_operation *)pos;

        memcpy(&eht_oper->basic_mcs_nss, eht_mcs_nss, sizeof(*eht_mcs_nss));
        eht_oper->params |= IEEE80211_EHT_OPER_INFO_PRESENT;
        pos += eht_oper_len;

        eht_oper_info =
                (struct ieee80211_eht_operation_info *)eht_oper->optional;

        eht_oper_info->ccfs0 =
                ieee80211_frequency_to_channel(chandef->center_freq1);
        if (chandef->center_freq2)
                eht_oper_info->ccfs1 =
                        ieee80211_frequency_to_channel(chandef->center_freq2);
        else
                eht_oper_info->ccfs1 = 0;

        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_320:
                chan_width = IEEE80211_EHT_OPER_CHAN_WIDTH_320MHZ;
                eht_oper_info->ccfs1 = eht_oper_info->ccfs0;
                if (chandef->chan->center_freq < chandef->center_freq1)
                        eht_oper_info->ccfs0 -= 16;
                else
                        eht_oper_info->ccfs0 += 16;
                break;
        case NL80211_CHAN_WIDTH_160:
                eht_oper_info->ccfs1 = eht_oper_info->ccfs0;
                if (chandef->chan->center_freq < chandef->center_freq1)
                        eht_oper_info->ccfs0 -= 8;
                else
                        eht_oper_info->ccfs0 += 8;
                fallthrough;
        case NL80211_CHAN_WIDTH_80P80:
                chan_width = IEEE80211_EHT_OPER_CHAN_WIDTH_160MHZ;
                break;
        case NL80211_CHAN_WIDTH_80:
                chan_width = IEEE80211_EHT_OPER_CHAN_WIDTH_80MHZ;
                break;
        case NL80211_CHAN_WIDTH_40:
                chan_width = IEEE80211_EHT_OPER_CHAN_WIDTH_40MHZ;
                break;
        default:
                chan_width = IEEE80211_EHT_OPER_CHAN_WIDTH_20MHZ;
                break;
        }
        eht_oper_info->control = chan_width;
        pos += eht_oper_info_len;

        /* TODO: eht_oper_info->optional */

        return pos;
}

bool ieee80211_chandef_ht_oper(const struct ieee80211_ht_operation *ht_oper,
                               struct cfg80211_chan_def *chandef)
{
        enum nl80211_channel_type channel_type;

        if (!ht_oper)
                return false;

        switch (ht_oper->ht_param & IEEE80211_HT_PARAM_CHA_SEC_OFFSET) {
        case IEEE80211_HT_PARAM_CHA_SEC_NONE:
                channel_type = NL80211_CHAN_HT20;
                break;
        case IEEE80211_HT_PARAM_CHA_SEC_ABOVE:
                channel_type = NL80211_CHAN_HT40PLUS;
                break;
        case IEEE80211_HT_PARAM_CHA_SEC_BELOW:
                channel_type = NL80211_CHAN_HT40MINUS;
                break;
        default:
                return false;
        }

        cfg80211_chandef_create(chandef, chandef->chan, channel_type);
        return true;
}

bool ieee80211_chandef_vht_oper(struct ieee80211_hw *hw, u32 vht_cap_info,
                                const struct ieee80211_vht_operation *oper,
                                const struct ieee80211_ht_operation *htop,
                                struct cfg80211_chan_def *chandef)
{
        struct cfg80211_chan_def new = *chandef;
        int cf0, cf1;
        int ccfs0, ccfs1, ccfs2;
        int ccf0, ccf1;
        u32 vht_cap;
        bool support_80_80 = false;
        bool support_160 = false;
        u8 ext_nss_bw_supp = u32_get_bits(vht_cap_info,
                                          IEEE80211_VHT_CAP_EXT_NSS_BW_MASK);
        u8 supp_chwidth = u32_get_bits(vht_cap_info,
                                       IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK);

        if (!oper || !htop)
                return false;

        vht_cap = hw->wiphy->bands[chandef->chan->band]->vht_cap.cap;
        support_160 = (vht_cap & (IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK |
                                  IEEE80211_VHT_CAP_EXT_NSS_BW_MASK));
        support_80_80 = ((vht_cap &
                         IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ) ||
                        (vht_cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ &&
                         vht_cap & IEEE80211_VHT_CAP_EXT_NSS_BW_MASK) ||
                        ((vht_cap & IEEE80211_VHT_CAP_EXT_NSS_BW_MASK) >>
                                    IEEE80211_VHT_CAP_EXT_NSS_BW_SHIFT > 1));
        ccfs0 = oper->center_freq_seg0_idx;
        ccfs1 = oper->center_freq_seg1_idx;
        ccfs2 = (le16_to_cpu(htop->operation_mode) &
                                IEEE80211_HT_OP_MODE_CCFS2_MASK)
                        >> IEEE80211_HT_OP_MODE_CCFS2_SHIFT;

        ccf0 = ccfs0;

        /* if not supported, parse as though we didn't understand it */
        if (!ieee80211_hw_check(hw, SUPPORTS_VHT_EXT_NSS_BW))
                ext_nss_bw_supp = 0;

        /*
         * Cf. IEEE 802.11 Table 9-250
         *
         * We really just consider that because it's inefficient to connect
         * at a higher bandwidth than we'll actually be able to use.
         */
        switch ((supp_chwidth << 4) | ext_nss_bw_supp) {
        default:
        case 0x00:
                ccf1 = 0;
                support_160 = false;
                support_80_80 = false;
                break;
        case 0x01:
                support_80_80 = false;
                fallthrough;
        case 0x02:
        case 0x03:
                ccf1 = ccfs2;
                break;
        case 0x10:
                ccf1 = ccfs1;
                break;
        case 0x11:
        case 0x12:
                if (!ccfs1)
                        ccf1 = ccfs2;
                else
                        ccf1 = ccfs1;
                break;
        case 0x13:
        case 0x20:
        case 0x23:
                ccf1 = ccfs1;
                break;
        }

        cf0 = ieee80211_channel_to_frequency(ccf0, chandef->chan->band);
        cf1 = ieee80211_channel_to_frequency(ccf1, chandef->chan->band);

        switch (oper->chan_width) {
        case IEEE80211_VHT_CHANWIDTH_USE_HT:
                /* just use HT information directly */
                break;
        case IEEE80211_VHT_CHANWIDTH_80MHZ:
                new.width = NL80211_CHAN_WIDTH_80;
                new.center_freq1 = cf0;
                /* If needed, adjust based on the newer interop workaround. */
                if (ccf1) {
                        unsigned int diff;

                        diff = abs(ccf1 - ccf0);
                        if ((diff == 8) && support_160) {
                                new.width = NL80211_CHAN_WIDTH_160;
                                new.center_freq1 = cf1;
                        } else if ((diff > 8) && support_80_80) {
                                new.width = NL80211_CHAN_WIDTH_80P80;
                                new.center_freq2 = cf1;
                        }
                }
                break;
        case IEEE80211_VHT_CHANWIDTH_160MHZ:
                /* deprecated encoding */
                new.width = NL80211_CHAN_WIDTH_160;
                new.center_freq1 = cf0;
                break;
        case IEEE80211_VHT_CHANWIDTH_80P80MHZ:
                /* deprecated encoding */
                new.width = NL80211_CHAN_WIDTH_80P80;
                new.center_freq1 = cf0;
                new.center_freq2 = cf1;
                break;
        default:
                return false;
        }

        if (!cfg80211_chandef_valid(&new))
                return false;

        *chandef = new;
        return true;
}

void ieee80211_chandef_eht_oper(const struct ieee80211_eht_operation_info *info,
                                struct cfg80211_chan_def *chandef)
{
        chandef->center_freq1 =
                ieee80211_channel_to_frequency(info->ccfs0,
                                               chandef->chan->band);

        switch (u8_get_bits(info->control,
                            IEEE80211_EHT_OPER_CHAN_WIDTH)) {
        case IEEE80211_EHT_OPER_CHAN_WIDTH_20MHZ:
                chandef->width = NL80211_CHAN_WIDTH_20;
                break;
        case IEEE80211_EHT_OPER_CHAN_WIDTH_40MHZ:
                chandef->width = NL80211_CHAN_WIDTH_40;
                break;
        case IEEE80211_EHT_OPER_CHAN_WIDTH_80MHZ:
                chandef->width = NL80211_CHAN_WIDTH_80;
                break;
        case IEEE80211_EHT_OPER_CHAN_WIDTH_160MHZ:
                chandef->width = NL80211_CHAN_WIDTH_160;
                chandef->center_freq1 =
                        ieee80211_channel_to_frequency(info->ccfs1,
                                                       chandef->chan->band);
                break;
        case IEEE80211_EHT_OPER_CHAN_WIDTH_320MHZ:
                chandef->width = NL80211_CHAN_WIDTH_320;
                chandef->center_freq1 =
                        ieee80211_channel_to_frequency(info->ccfs1,
                                                       chandef->chan->band);
                break;
        }
}

bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_local *local,
                                    const struct ieee80211_he_operation *he_oper,
                                    const struct ieee80211_eht_operation *eht_oper,
                                    struct cfg80211_chan_def *chandef)
{
        struct cfg80211_chan_def he_chandef = *chandef;
        const struct ieee80211_he_6ghz_oper *he_6ghz_oper;
        u32 freq;

        if (chandef->chan->band != NL80211_BAND_6GHZ)
                return true;

        if (!he_oper)
                return false;

        he_6ghz_oper = ieee80211_he_6ghz_oper(he_oper);
        if (!he_6ghz_oper)
                return false;

        /*
         * The EHT operation IE does not contain the primary channel so the
         * primary channel frequency should be taken from the 6 GHz operation
         * information.
         */
        freq = ieee80211_channel_to_frequency(he_6ghz_oper->primary,
                                              NL80211_BAND_6GHZ);
        he_chandef.chan = ieee80211_get_channel(local->hw.wiphy, freq);

        if (!he_chandef.chan)
                return false;

        if (!eht_oper ||
            !(eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT)) {
                switch (u8_get_bits(he_6ghz_oper->control,
                                    IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH)) {
                case IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_20MHZ:
                        he_chandef.width = NL80211_CHAN_WIDTH_20;
                        break;
                case IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_40MHZ:
                        he_chandef.width = NL80211_CHAN_WIDTH_40;
                        break;
                case IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_80MHZ:
                        he_chandef.width = NL80211_CHAN_WIDTH_80;
                        break;
                case IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_160MHZ:
                        he_chandef.width = NL80211_CHAN_WIDTH_80;
                        if (!he_6ghz_oper->ccfs1)
                                break;
                        if (abs(he_6ghz_oper->ccfs1 - he_6ghz_oper->ccfs0) == 8)
                                he_chandef.width = NL80211_CHAN_WIDTH_160;
                        else
                                he_chandef.width = NL80211_CHAN_WIDTH_80P80;
                        break;
                }

                if (he_chandef.width == NL80211_CHAN_WIDTH_160) {
                        he_chandef.center_freq1 =
                                ieee80211_channel_to_frequency(he_6ghz_oper->ccfs1,
                                                               NL80211_BAND_6GHZ);
                } else {
                        he_chandef.center_freq1 =
                                ieee80211_channel_to_frequency(he_6ghz_oper->ccfs0,
                                                               NL80211_BAND_6GHZ);
                        he_chandef.center_freq2 =
                                ieee80211_channel_to_frequency(he_6ghz_oper->ccfs1,
                                                               NL80211_BAND_6GHZ);
                }
        } else {
                ieee80211_chandef_eht_oper((const void *)eht_oper->optional,
                                           &he_chandef);
                he_chandef.punctured =
                        ieee80211_eht_oper_dis_subchan_bitmap(eht_oper);
        }

        if (!cfg80211_chandef_valid(&he_chandef))
                return false;

        *chandef = he_chandef;

        return true;
}

bool ieee80211_chandef_s1g_oper(const struct ieee80211_s1g_oper_ie *oper,
                                struct cfg80211_chan_def *chandef)
{
        u32 oper_freq;

        if (!oper)
                return false;

        switch (FIELD_GET(S1G_OPER_CH_WIDTH_OPER, oper->ch_width)) {
        case IEEE80211_S1G_CHANWIDTH_1MHZ:
                chandef->width = NL80211_CHAN_WIDTH_1;
                break;
        case IEEE80211_S1G_CHANWIDTH_2MHZ:
                chandef->width = NL80211_CHAN_WIDTH_2;
                break;
        case IEEE80211_S1G_CHANWIDTH_4MHZ:
                chandef->width = NL80211_CHAN_WIDTH_4;
                break;
        case IEEE80211_S1G_CHANWIDTH_8MHZ:
                chandef->width = NL80211_CHAN_WIDTH_8;
                break;
        case IEEE80211_S1G_CHANWIDTH_16MHZ:
                chandef->width = NL80211_CHAN_WIDTH_16;
                break;
        default:
                return false;
        }

        oper_freq = ieee80211_channel_to_freq_khz(oper->oper_ch,
                                                  NL80211_BAND_S1GHZ);
        chandef->center_freq1 = KHZ_TO_MHZ(oper_freq);
        chandef->freq1_offset = oper_freq % 1000;

        return true;
}

int ieee80211_put_srates_elem(struct sk_buff *skb,
                              const struct ieee80211_supported_band *sband,
                              u32 basic_rates, u32 rate_flags, u32 masked_rates,
                              u8 element_id)
{
        u8 i, rates, skip;

        rates = 0;
        for (i = 0; i < sband->n_bitrates; i++) {
                if ((rate_flags & sband->bitrates[i].flags) != rate_flags)
                        continue;
                if (masked_rates & BIT(i))
                        continue;
                rates++;
        }

        if (element_id == WLAN_EID_SUPP_RATES) {
                rates = min_t(u8, rates, 8);
                skip = 0;
        } else {
                skip = 8;
                if (rates <= skip)
                        return 0;
                rates -= skip;
        }

        if (skb_tailroom(skb) < rates + 2)
                return -ENOBUFS;

        skb_put_u8(skb, element_id);
        skb_put_u8(skb, rates);

        for (i = 0; i < sband->n_bitrates && rates; i++) {
                int rate;
                u8 basic;

                if ((rate_flags & sband->bitrates[i].flags) != rate_flags)
                        continue;
                if (masked_rates & BIT(i))
                        continue;

                if (skip > 0) {
                        skip--;
                        continue;
                }

                basic = basic_rates & BIT(i) ? 0x80 : 0;

                rate = DIV_ROUND_UP(sband->bitrates[i].bitrate, 5);
                skb_put_u8(skb, basic | (u8)rate);
                rates--;
        }

        WARN(rates > 0, "rates confused: rates:%d, element:%d\n",
             rates, element_id);

        return 0;
}

int ieee80211_ave_rssi(struct ieee80211_vif *vif)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);

        if (WARN_ON_ONCE(sdata->vif.type != NL80211_IFTYPE_STATION))
                return 0;

        return -ewma_beacon_signal_read(&sdata->deflink.u.mgd.ave_beacon_signal);
}
EXPORT_SYMBOL_GPL(ieee80211_ave_rssi);

u8 ieee80211_mcs_to_chains(const struct ieee80211_mcs_info *mcs)
{
        if (!mcs)
                return 1;

        /* TODO: consider rx_highest */

        if (mcs->rx_mask[3])
                return 4;
        if (mcs->rx_mask[2])
                return 3;
        if (mcs->rx_mask[1])
                return 2;
        return 1;
}

/**
 * ieee80211_calculate_rx_timestamp - calculate timestamp in frame
 * @local: mac80211 hw info struct
 * @status: RX status
 * @mpdu_len: total MPDU length (including FCS)
 * @mpdu_offset: offset into MPDU to calculate timestamp at
 *
 * This function calculates the RX timestamp at the given MPDU offset, taking
 * into account what the RX timestamp was. An offset of 0 will just normalize
 * the timestamp to TSF at beginning of MPDU reception.
 *
 * Returns: the calculated timestamp
 */
u64 ieee80211_calculate_rx_timestamp(struct ieee80211_local *local,
                                     struct ieee80211_rx_status *status,
                                     unsigned int mpdu_len,
                                     unsigned int mpdu_offset)
{
        u64 ts = status->mactime;
        bool mactime_plcp_start;
        struct rate_info ri;
        u16 rate;
        u8 n_ltf;

        if (WARN_ON(!ieee80211_have_rx_timestamp(status)))
                return 0;

        mactime_plcp_start = (status->flag & RX_FLAG_MACTIME) ==
                                RX_FLAG_MACTIME_PLCP_START;

        memset(&ri, 0, sizeof(ri));

        ri.bw = status->bw;

        /* Fill cfg80211 rate info */
        switch (status->encoding) {
        case RX_ENC_EHT:
                ri.flags |= RATE_INFO_FLAGS_EHT_MCS;
                ri.mcs = status->rate_idx;
                ri.nss = status->nss;
                ri.eht_ru_alloc = status->eht.ru;
                if (status->enc_flags & RX_ENC_FLAG_SHORT_GI)
                        ri.flags |= RATE_INFO_FLAGS_SHORT_GI;
                /* TODO/FIXME: is this right? handle other PPDUs */
                if (mactime_plcp_start) {
                        mpdu_offset += 2;
                        ts += 36;
                }
                break;
        case RX_ENC_HE:
                ri.flags |= RATE_INFO_FLAGS_HE_MCS;
                ri.mcs = status->rate_idx;
                ri.nss = status->nss;
                ri.he_ru_alloc = status->he_ru;
                if (status->enc_flags & RX_ENC_FLAG_SHORT_GI)
                        ri.flags |= RATE_INFO_FLAGS_SHORT_GI;

                /*
                 * See P802.11ax_D6.0, section 27.3.4 for
                 * VHT PPDU format.
                 */
                if (mactime_plcp_start) {
                        mpdu_offset += 2;
                        ts += 36;

                        /*
                         * TODO:
                         * For HE MU PPDU, add the HE-SIG-B.
                         * For HE ER PPDU, add 8us for the HE-SIG-A.
                         * For HE TB PPDU, add 4us for the HE-STF.
                         * Add the HE-LTF durations - variable.
                         */
                }

                break;
        case RX_ENC_HT:
                ri.mcs = status->rate_idx;
                ri.flags |= RATE_INFO_FLAGS_MCS;
                if (status->enc_flags & RX_ENC_FLAG_SHORT_GI)
                        ri.flags |= RATE_INFO_FLAGS_SHORT_GI;

                /*
                 * See P802.11REVmd_D3.0, section 19.3.2 for
                 * HT PPDU format.
                 */
                if (mactime_plcp_start) {
                        mpdu_offset += 2;
                        if (status->enc_flags & RX_ENC_FLAG_HT_GF)
                                ts += 24;
                        else
                                ts += 32;

                        /*
                         * Add Data HT-LTFs per streams
                         * TODO: add Extension HT-LTFs, 4us per LTF
                         */
                        n_ltf = ((ri.mcs >> 3) & 3) + 1;
                        n_ltf = n_ltf == 3 ? 4 : n_ltf;
                        ts += n_ltf * 4;
                }

                break;
        case RX_ENC_VHT:
                ri.flags |= RATE_INFO_FLAGS_VHT_MCS;
                ri.mcs = status->rate_idx;
                ri.nss = status->nss;
                if (status->enc_flags & RX_ENC_FLAG_SHORT_GI)
                        ri.flags |= RATE_INFO_FLAGS_SHORT_GI;

                /*
                 * See P802.11REVmd_D3.0, section 21.3.2 for
                 * VHT PPDU format.
                 */
                if (mactime_plcp_start) {
                        mpdu_offset += 2;
                        ts += 36;

                        /*
                         * Add VHT-LTFs per streams
                         */
                        n_ltf = (ri.nss != 1) && (ri.nss % 2) ?
                                ri.nss + 1 : ri.nss;
                        ts += 4 * n_ltf;
                }

                break;
        default:
                WARN_ON(1);
                fallthrough;
        case RX_ENC_LEGACY: {
                struct ieee80211_supported_band *sband;

                sband = local->hw.wiphy->bands[status->band];
                ri.legacy = sband->bitrates[status->rate_idx].bitrate;

                if (mactime_plcp_start) {
                        if (status->band == NL80211_BAND_5GHZ) {
                                ts += 20;
                                mpdu_offset += 2;
                        } else if (status->enc_flags & RX_ENC_FLAG_SHORTPRE) {
                                ts += 96;
                        } else {
                                ts += 192;
                        }
                }
                break;
                }
        }

        rate = cfg80211_calculate_bitrate(&ri);
        if (WARN_ONCE(!rate,
                      "Invalid bitrate: flags=0x%llx, idx=%d, vht_nss=%d\n",
                      (unsigned long long)status->flag, status->rate_idx,
                      status->nss))
                return 0;

        /* rewind from end of MPDU */
        if ((status->flag & RX_FLAG_MACTIME) == RX_FLAG_MACTIME_END)
                ts -= mpdu_len * 8 * 10 / rate;

        ts += mpdu_offset * 8 * 10 / rate;

        return ts;
}

/* Cancel CAC for the interfaces under the specified @local. If @ctx is
 * also provided, only the interfaces using that ctx will be canceled.
 */
void ieee80211_dfs_cac_cancel(struct ieee80211_local *local,
                              struct ieee80211_chanctx *ctx)
{
        struct ieee80211_sub_if_data *sdata;
        struct cfg80211_chan_def chandef;
        struct ieee80211_link_data *link;
        struct ieee80211_chanctx_conf *chanctx_conf;
        unsigned int link_id;

        lockdep_assert_wiphy(local->hw.wiphy);

        list_for_each_entry(sdata, &local->interfaces, list) {
                for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS;
                     link_id++) {
                        link = sdata_dereference(sdata->link[link_id],
                                                 sdata);
                        if (!link)
                                continue;

                        chanctx_conf = sdata_dereference(link->conf->chanctx_conf,
                                                         sdata);
                        if (ctx && &ctx->conf != chanctx_conf)
                                continue;

                        wiphy_delayed_work_cancel(local->hw.wiphy,
                                                  &link->dfs_cac_timer_work);

                        if (!sdata->wdev.links[link_id].cac_started)
                                continue;

                        chandef = link->conf->chanreq.oper;
                        ieee80211_link_release_channel(link);
                        cfg80211_cac_event(sdata->dev, &chandef,
                                           NL80211_RADAR_CAC_ABORTED,
                                           GFP_KERNEL, link_id);
                }
        }
}

void ieee80211_dfs_radar_detected_work(struct wiphy *wiphy,
                                       struct wiphy_work *work)
{
        struct ieee80211_local *local =
                container_of(work, struct ieee80211_local, radar_detected_work);
        struct cfg80211_chan_def chandef;
        struct ieee80211_chanctx *ctx;

        lockdep_assert_wiphy(local->hw.wiphy);

        list_for_each_entry(ctx, &local->chanctx_list, list) {
                if (ctx->replace_state == IEEE80211_CHANCTX_REPLACES_OTHER)
                        continue;

                if (!ctx->radar_detected)
                        continue;

                ctx->radar_detected = false;

                chandef = ctx->conf.def;

                ieee80211_dfs_cac_cancel(local, ctx);
                cfg80211_radar_event(local->hw.wiphy, &chandef, GFP_KERNEL);
        }
}

static void
ieee80211_radar_mark_chan_ctx_iterator(struct ieee80211_hw *hw,
                                       struct ieee80211_chanctx_conf *chanctx_conf,
                                       void *data)
{
        struct ieee80211_chanctx *ctx =
                container_of(chanctx_conf, struct ieee80211_chanctx,
                             conf);

        if (ctx->replace_state == IEEE80211_CHANCTX_REPLACES_OTHER)
                return;

        if (data && data != chanctx_conf)
                return;

        ctx->radar_detected = true;
}

void ieee80211_radar_detected(struct ieee80211_hw *hw,
                              struct ieee80211_chanctx_conf *chanctx_conf)
{
        struct ieee80211_local *local = hw_to_local(hw);

        trace_api_radar_detected(local);

        ieee80211_iter_chan_contexts_atomic(hw, ieee80211_radar_mark_chan_ctx_iterator,
                                            chanctx_conf);

        wiphy_work_queue(hw->wiphy, &local->radar_detected_work);
}
EXPORT_SYMBOL(ieee80211_radar_detected);

void ieee80211_chandef_downgrade(struct cfg80211_chan_def *c,
                                 struct ieee80211_conn_settings *conn)
{
        enum nl80211_chan_width new_primary_width;
        struct ieee80211_conn_settings _ignored = {};

        /* allow passing NULL if caller doesn't care */
        if (!conn)
                conn = &_ignored;

again:
        /* no-HT indicates nothing to do */
        new_primary_width = NL80211_CHAN_WIDTH_20_NOHT;

        switch (c->width) {
        default:
        case NL80211_CHAN_WIDTH_20_NOHT:
                WARN_ON_ONCE(1);
                fallthrough;
        case NL80211_CHAN_WIDTH_20:
                c->width = NL80211_CHAN_WIDTH_20_NOHT;
                conn->mode = IEEE80211_CONN_MODE_LEGACY;
                conn->bw_limit = IEEE80211_CONN_BW_LIMIT_20;
                c->punctured = 0;
                break;
        case NL80211_CHAN_WIDTH_40:
                c->width = NL80211_CHAN_WIDTH_20;
                c->center_freq1 = c->chan->center_freq;
                if (conn->mode == IEEE80211_CONN_MODE_VHT)
                        conn->mode = IEEE80211_CONN_MODE_HT;
                conn->bw_limit = IEEE80211_CONN_BW_LIMIT_20;
                c->punctured = 0;
                break;
        case NL80211_CHAN_WIDTH_80:
                new_primary_width = NL80211_CHAN_WIDTH_40;
                if (conn->mode == IEEE80211_CONN_MODE_VHT)
                        conn->mode = IEEE80211_CONN_MODE_HT;
                conn->bw_limit = IEEE80211_CONN_BW_LIMIT_40;
                break;
        case NL80211_CHAN_WIDTH_80P80:
                c->center_freq2 = 0;
                c->width = NL80211_CHAN_WIDTH_80;
                conn->bw_limit = IEEE80211_CONN_BW_LIMIT_80;
                break;
        case NL80211_CHAN_WIDTH_160:
                new_primary_width = NL80211_CHAN_WIDTH_80;
                conn->bw_limit = IEEE80211_CONN_BW_LIMIT_80;
                break;
        case NL80211_CHAN_WIDTH_320:
                new_primary_width = NL80211_CHAN_WIDTH_160;
                conn->bw_limit = IEEE80211_CONN_BW_LIMIT_160;
                break;
        case NL80211_CHAN_WIDTH_1:
        case NL80211_CHAN_WIDTH_2:
        case NL80211_CHAN_WIDTH_4:
        case NL80211_CHAN_WIDTH_8:
        case NL80211_CHAN_WIDTH_16:
                WARN_ON_ONCE(1);
                /* keep c->width */
                conn->mode = IEEE80211_CONN_MODE_S1G;
                conn->bw_limit = IEEE80211_CONN_BW_LIMIT_20;
                break;
        case NL80211_CHAN_WIDTH_5:
        case NL80211_CHAN_WIDTH_10:
                WARN_ON_ONCE(1);
                /* keep c->width */
                conn->mode = IEEE80211_CONN_MODE_LEGACY;
                conn->bw_limit = IEEE80211_CONN_BW_LIMIT_20;
                break;
        }

        if (new_primary_width != NL80211_CHAN_WIDTH_20_NOHT) {
                c->center_freq1 = cfg80211_chandef_primary(c, new_primary_width,
                                                           &c->punctured);
                c->width = new_primary_width;
        }

        /*
         * With an 80 MHz channel, we might have the puncturing in the primary
         * 40 Mhz channel, but that's not valid when downgraded to 40 MHz width.
         * In that case, downgrade again.
         */
        if (!cfg80211_chandef_valid(c) && c->punctured)
                goto again;

        WARN_ON_ONCE(!cfg80211_chandef_valid(c));
}

/*
 * Returns true if smps_mode_new is strictly more restrictive than
 * smps_mode_old.
 */
bool ieee80211_smps_is_restrictive(enum ieee80211_smps_mode smps_mode_old,
                                   enum ieee80211_smps_mode smps_mode_new)
{
        if (WARN_ON_ONCE(smps_mode_old == IEEE80211_SMPS_AUTOMATIC ||
                         smps_mode_new == IEEE80211_SMPS_AUTOMATIC))
                return false;

        switch (smps_mode_old) {
        case IEEE80211_SMPS_STATIC:
                return false;
        case IEEE80211_SMPS_DYNAMIC:
                return smps_mode_new == IEEE80211_SMPS_STATIC;
        case IEEE80211_SMPS_OFF:
                return smps_mode_new != IEEE80211_SMPS_OFF;
        default:
                WARN_ON(1);
        }

        return false;
}

int ieee80211_send_action_csa(struct ieee80211_sub_if_data *sdata,
                              struct cfg80211_csa_settings *csa_settings)
{
        struct sk_buff *skb;
        struct ieee80211_mgmt *mgmt;
        struct ieee80211_local *local = sdata->local;
        int freq;
        int hdr_len = offsetofend(struct ieee80211_mgmt,
                                  u.action.u.chan_switch);
        u8 *pos;

        if (sdata->vif.type != NL80211_IFTYPE_ADHOC &&
            sdata->vif.type != NL80211_IFTYPE_MESH_POINT)
                return -EOPNOTSUPP;

        skb = dev_alloc_skb(local->tx_headroom + hdr_len +
                            5 + /* channel switch announcement element */
                            3 + /* secondary channel offset element */
                            5 + /* wide bandwidth channel switch announcement */
                            8); /* mesh channel switch parameters element */
        if (!skb)
                return -ENOMEM;

        skb_reserve(skb, local->tx_headroom);
        mgmt = skb_put_zero(skb, hdr_len);
        mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
                                          IEEE80211_STYPE_ACTION);

        eth_broadcast_addr(mgmt->da);
        memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN);
        if (ieee80211_vif_is_mesh(&sdata->vif)) {
                memcpy(mgmt->bssid, sdata->vif.addr, ETH_ALEN);
        } else {
                struct ieee80211_if_ibss *ifibss = &sdata->u.ibss;
                memcpy(mgmt->bssid, ifibss->bssid, ETH_ALEN);
        }
        mgmt->u.action.category = WLAN_CATEGORY_SPECTRUM_MGMT;
        mgmt->u.action.u.chan_switch.action_code = WLAN_ACTION_SPCT_CHL_SWITCH;
        pos = skb_put(skb, 5);
        *pos++ = WLAN_EID_CHANNEL_SWITCH;                        /* EID */
        *pos++ = 3;                                                /* IE length */
        *pos++ = csa_settings->block_tx ? 1 : 0;                /* CSA mode */
        freq = csa_settings->chandef.chan->center_freq;
        *pos++ = ieee80211_frequency_to_channel(freq);                /* channel */
        *pos++ = csa_settings->count;                                /* count */

        if (csa_settings->chandef.width == NL80211_CHAN_WIDTH_40) {
                enum nl80211_channel_type ch_type;

                skb_put(skb, 3);
                *pos++ = WLAN_EID_SECONDARY_CHANNEL_OFFSET;        /* EID */
                *pos++ = 1;                                        /* IE length */
                ch_type = cfg80211_get_chandef_type(&csa_settings->chandef);
                if (ch_type == NL80211_CHAN_HT40PLUS)
                        *pos++ = IEEE80211_HT_PARAM_CHA_SEC_ABOVE;
                else
                        *pos++ = IEEE80211_HT_PARAM_CHA_SEC_BELOW;
        }

        if (ieee80211_vif_is_mesh(&sdata->vif)) {
                struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;

                skb_put(skb, 8);
                *pos++ = WLAN_EID_CHAN_SWITCH_PARAM;                /* EID */
                *pos++ = 6;                                        /* IE length */
                *pos++ = sdata->u.mesh.mshcfg.dot11MeshTTL;        /* Mesh TTL */
                *pos = 0x00;        /* Mesh Flag: Tx Restrict, Initiator, Reason */
                *pos |= WLAN_EID_CHAN_SWITCH_PARAM_INITIATOR;
                *pos++ |= csa_settings->block_tx ?
                          WLAN_EID_CHAN_SWITCH_PARAM_TX_RESTRICT : 0x00;
                put_unaligned_le16(WLAN_REASON_MESH_CHAN, pos); /* Reason Cd */
                pos += 2;
                put_unaligned_le16(ifmsh->pre_value, pos);/* Precedence Value */
                pos += 2;
        }

        if (csa_settings->chandef.width == NL80211_CHAN_WIDTH_80 ||
            csa_settings->chandef.width == NL80211_CHAN_WIDTH_80P80 ||
            csa_settings->chandef.width == NL80211_CHAN_WIDTH_160) {
                skb_put(skb, 5);
                ieee80211_ie_build_wide_bw_cs(pos, &csa_settings->chandef);
        }

        ieee80211_tx_skb(sdata, skb);
        return 0;
}

static bool
ieee80211_extend_noa_desc(struct ieee80211_noa_data *data, u32 tsf, int i)
{
        s32 end = data->desc[i].start + data->desc[i].duration - (tsf + 1);
        int skip;

        if (end > 0)
                return false;

        /* One shot NOA  */
        if (data->count[i] == 1)
                return false;

        if (data->desc[i].interval == 0)
                return false;

        /* End time is in the past, check for repetitions */
        skip = DIV_ROUND_UP(-end, data->desc[i].interval);
        if (data->count[i] < 255) {
                if (data->count[i] <= skip) {
                        data->count[i] = 0;
                        return false;
                }

                data->count[i] -= skip;
        }

        data->desc[i].start += skip * data->desc[i].interval;

        return true;
}

static bool
ieee80211_extend_absent_time(struct ieee80211_noa_data *data, u32 tsf,
                             s32 *offset)
{
        bool ret = false;
        int i;

        for (i = 0; i < IEEE80211_P2P_NOA_DESC_MAX; i++) {
                s32 cur;

                if (!data->count[i])
                        continue;

                if (ieee80211_extend_noa_desc(data, tsf + *offset, i))
                        ret = true;

                cur = data->desc[i].start - tsf;
                if (cur > *offset)
                        continue;

                cur = data->desc[i].start + data->desc[i].duration - tsf;
                if (cur > *offset)
                        *offset = cur;
        }

        return ret;
}

static u32
ieee80211_get_noa_absent_time(struct ieee80211_noa_data *data, u32 tsf)
{
        s32 offset = 0;
        int tries = 0;
        /*
         * arbitrary limit, used to avoid infinite loops when combined NoA
         * descriptors cover the full time period.
         */
        int max_tries = 5;

        ieee80211_extend_absent_time(data, tsf, &offset);
        do {
                if (!ieee80211_extend_absent_time(data, tsf, &offset))
                        break;

                tries++;
        } while (tries < max_tries);

        return offset;
}

void ieee80211_update_p2p_noa(struct ieee80211_noa_data *data, u32 tsf)
{
        u32 next_offset = BIT(31) - 1;
        int i;

        data->absent = 0;
        data->has_next_tsf = false;
        for (i = 0; i < IEEE80211_P2P_NOA_DESC_MAX; i++) {
                s32 start;

                if (!data->count[i])
                        continue;

                ieee80211_extend_noa_desc(data, tsf, i);
                start = data->desc[i].start - tsf;
                if (start <= 0)
                        data->absent |= BIT(i);

                if (next_offset > start)
                        next_offset = start;

                data->has_next_tsf = true;
        }

        if (data->absent)
                next_offset = ieee80211_get_noa_absent_time(data, tsf);

        data->next_tsf = tsf + next_offset;
}
EXPORT_SYMBOL(ieee80211_update_p2p_noa);

int ieee80211_parse_p2p_noa(const struct ieee80211_p2p_noa_attr *attr,
                            struct ieee80211_noa_data *data, u32 tsf)
{
        int ret = 0;
        int i;

        memset(data, 0, sizeof(*data));

        for (i = 0; i < IEEE80211_P2P_NOA_DESC_MAX; i++) {
                const struct ieee80211_p2p_noa_desc *desc = &attr->desc[i];

                if (!desc->count || !desc->duration)
                        continue;

                data->count[i] = desc->count;
                data->desc[i].start = le32_to_cpu(desc->start_time);
                data->desc[i].duration = le32_to_cpu(desc->duration);
                data->desc[i].interval = le32_to_cpu(desc->interval);

                if (data->count[i] > 1 &&
                    data->desc[i].interval < data->desc[i].duration)
                        continue;

                ieee80211_extend_noa_desc(data, tsf, i);
                ret++;
        }

        if (ret)
                ieee80211_update_p2p_noa(data, tsf);

        return ret;
}
EXPORT_SYMBOL(ieee80211_parse_p2p_noa);

void ieee80211_recalc_dtim(struct ieee80211_local *local,
                           struct ieee80211_sub_if_data *sdata)
{
        u64 tsf = drv_get_tsf(local, sdata);
        u64 dtim_count = 0;
        u16 beacon_int = sdata->vif.bss_conf.beacon_int * 1024;
        u8 dtim_period = sdata->vif.bss_conf.dtim_period;
        struct ps_data *ps;
        u8 bcns_from_dtim;

        if (tsf == -1ULL || !beacon_int || !dtim_period)
                return;

        if (sdata->vif.type == NL80211_IFTYPE_AP ||
            sdata->vif.type == NL80211_IFTYPE_AP_VLAN) {
                if (!sdata->bss)
                        return;

                ps = &sdata->bss->ps;
        } else if (ieee80211_vif_is_mesh(&sdata->vif)) {
                ps = &sdata->u.mesh.ps;
        } else {
                return;
        }

        /*
         * actually finds last dtim_count, mac80211 will update in
         * __beacon_add_tim().
         * dtim_count = dtim_period - (tsf / bcn_int) % dtim_period
         */
        do_div(tsf, beacon_int);
        bcns_from_dtim = do_div(tsf, dtim_period);
        /* just had a DTIM */
        if (!bcns_from_dtim)
                dtim_count = 0;
        else
                dtim_count = dtim_period - bcns_from_dtim;

        ps->dtim_count = dtim_count;
}

static u8 ieee80211_chanctx_radar_detect(struct ieee80211_local *local,
                                         struct ieee80211_chanctx *ctx)
{
        struct ieee80211_link_data *link;
        u8 radar_detect = 0;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (WARN_ON(ctx->replace_state == IEEE80211_CHANCTX_WILL_BE_REPLACED))
                return 0;

        list_for_each_entry(link, &ctx->reserved_links, reserved_chanctx_list)
                if (link->reserved_radar_required)
                        radar_detect |= BIT(link->reserved.oper.width);

        /*
         * An in-place reservation context should not have any assigned vifs
         * until it replaces the other context.
         */
        WARN_ON(ctx->replace_state == IEEE80211_CHANCTX_REPLACES_OTHER &&
                !list_empty(&ctx->assigned_links));

        list_for_each_entry(link, &ctx->assigned_links, assigned_chanctx_list) {
                if (!link->radar_required)
                        continue;

                radar_detect |=
                        BIT(link->conf->chanreq.oper.width);
        }

        return radar_detect;
}

static u32
__ieee80211_get_radio_mask(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_bss_conf *link_conf;
        struct ieee80211_chanctx_conf *conf;
        unsigned int link_id;
        u32 mask = 0;

        for_each_vif_active_link(&sdata->vif, link_conf, link_id) {
                conf = sdata_dereference(link_conf->chanctx_conf, sdata);
                if (!conf || conf->radio_idx < 0)
                        continue;

                mask |= BIT(conf->radio_idx);
        }

        return mask;
}

u32 ieee80211_get_radio_mask(struct wiphy *wiphy, struct net_device *dev)
{
        struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);

        return __ieee80211_get_radio_mask(sdata);
}

static bool
ieee80211_sdata_uses_radio(struct ieee80211_sub_if_data *sdata, int radio_idx)
{
        if (radio_idx < 0)
                return true;

        return __ieee80211_get_radio_mask(sdata) & BIT(radio_idx);
}

static int
ieee80211_fill_ifcomb_params(struct ieee80211_local *local,
                             struct iface_combination_params *params,
                             const struct cfg80211_chan_def *chandef,
                             struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_sub_if_data *sdata_iter;
        struct ieee80211_chanctx *ctx;
        int total = !!sdata;

        list_for_each_entry(ctx, &local->chanctx_list, list) {
                if (ctx->replace_state == IEEE80211_CHANCTX_WILL_BE_REPLACED)
                        continue;

                if (params->radio_idx >= 0 &&
                    ctx->conf.radio_idx != params->radio_idx)
                        continue;

                params->radar_detect |=
                        ieee80211_chanctx_radar_detect(local, ctx);

                if (chandef && ctx->mode != IEEE80211_CHANCTX_EXCLUSIVE &&
                    cfg80211_chandef_compatible(chandef, &ctx->conf.def))
                        continue;

                params->num_different_channels++;
        }

        list_for_each_entry(sdata_iter, &local->interfaces, list) {
                struct wireless_dev *wdev_iter;

                wdev_iter = &sdata_iter->wdev;

                if (sdata_iter == sdata ||
                    !ieee80211_sdata_running(sdata_iter) ||
                    cfg80211_iftype_allowed(local->hw.wiphy,
                                            wdev_iter->iftype, 0, 1))
                        continue;

                if (!ieee80211_sdata_uses_radio(sdata_iter, params->radio_idx))
                        continue;

                params->iftype_num[wdev_iter->iftype]++;
                total++;
        }

        return total;
}

int ieee80211_check_combinations(struct ieee80211_sub_if_data *sdata,
                                 const struct cfg80211_chan_def *chandef,
                                 enum ieee80211_chanctx_mode chanmode,
                                 u8 radar_detect, int radio_idx)
{
        bool shared = chanmode == IEEE80211_CHANCTX_SHARED;
        struct ieee80211_local *local = sdata->local;
        enum nl80211_iftype iftype = sdata->wdev.iftype;
        struct iface_combination_params params = {
                .radar_detect = radar_detect,
                .radio_idx = radio_idx,
        };
        int total;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (WARN_ON(hweight32(radar_detect) > 1))
                return -EINVAL;

        if (WARN_ON(chandef && chanmode == IEEE80211_CHANCTX_SHARED &&
                    !chandef->chan))
                return -EINVAL;

        if (WARN_ON(iftype >= NUM_NL80211_IFTYPES))
                return -EINVAL;

        if (sdata->vif.type == NL80211_IFTYPE_AP ||
            sdata->vif.type == NL80211_IFTYPE_MESH_POINT) {
                /*
                 * always passing this is harmless, since it'll be the
                 * same value that cfg80211 finds if it finds the same
                 * interface ... and that's always allowed
                 */
                params.new_beacon_int = sdata->vif.bss_conf.beacon_int;
        }

        /* Always allow software iftypes */
        if (cfg80211_iftype_allowed(local->hw.wiphy, iftype, 0, 1)) {
                if (radar_detect)
                        return -EINVAL;
                return 0;
        }

        if (chandef)
                params.num_different_channels = 1;

        if (iftype != NL80211_IFTYPE_UNSPECIFIED)
                params.iftype_num[iftype] = 1;

        total = ieee80211_fill_ifcomb_params(local, &params,
                                             shared ? chandef : NULL,
                                             sdata);
        if (total == 1 && !params.radar_detect)
                return 0;

        return cfg80211_check_combinations(local->hw.wiphy, &params);
}

static void
ieee80211_iter_max_chans(const struct ieee80211_iface_combination *c,
                         void *data)
{
        u32 *max_num_different_channels = data;

        *max_num_different_channels = max(*max_num_different_channels,
                                          c->num_different_channels);
}

int ieee80211_max_num_channels(struct ieee80211_local *local, int radio_idx)
{
        u32 max_num_different_channels = 1;
        int err;
        struct iface_combination_params params = {
                .radio_idx = radio_idx,
        };

        lockdep_assert_wiphy(local->hw.wiphy);

        ieee80211_fill_ifcomb_params(local, &params, NULL, NULL);

        err = cfg80211_iter_combinations(local->hw.wiphy, &params,
                                         ieee80211_iter_max_chans,
                                         &max_num_different_channels);
        if (err < 0)
                return err;

        return max_num_different_channels;
}

void ieee80211_add_s1g_capab_ie(struct ieee80211_sub_if_data *sdata,
                                struct ieee80211_sta_s1g_cap *caps,
                                struct sk_buff *skb)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_s1g_cap s1g_capab;
        u8 *pos;
        int i;

        if (WARN_ON(sdata->vif.type != NL80211_IFTYPE_STATION))
                return;

        if (!caps->s1g)
                return;

        memcpy(s1g_capab.capab_info, caps->cap, sizeof(caps->cap));
        memcpy(s1g_capab.supp_mcs_nss, caps->nss_mcs, sizeof(caps->nss_mcs));

        /* override the capability info */
        for (i = 0; i < sizeof(ifmgd->s1g_capa.capab_info); i++) {
                u8 mask = ifmgd->s1g_capa_mask.capab_info[i];

                s1g_capab.capab_info[i] &= ~mask;
                s1g_capab.capab_info[i] |= ifmgd->s1g_capa.capab_info[i] & mask;
        }

        /* then MCS and NSS set */
        for (i = 0; i < sizeof(ifmgd->s1g_capa.supp_mcs_nss); i++) {
                u8 mask = ifmgd->s1g_capa_mask.supp_mcs_nss[i];

                s1g_capab.supp_mcs_nss[i] &= ~mask;
                s1g_capab.supp_mcs_nss[i] |=
                        ifmgd->s1g_capa.supp_mcs_nss[i] & mask;
        }

        pos = skb_put(skb, 2 + sizeof(s1g_capab));
        *pos++ = WLAN_EID_S1G_CAPABILITIES;
        *pos++ = sizeof(s1g_capab);

        memcpy(pos, &s1g_capab, sizeof(s1g_capab));
}

void ieee80211_add_aid_request_ie(struct ieee80211_sub_if_data *sdata,
                                  struct sk_buff *skb)
{
        u8 *pos = skb_put(skb, 3);

        *pos++ = WLAN_EID_AID_REQUEST;
        *pos++ = 1;
        *pos++ = 0;
}

u8 *ieee80211_add_wmm_info_ie(u8 *buf, u8 qosinfo)
{
        *buf++ = WLAN_EID_VENDOR_SPECIFIC;
        *buf++ = 7; /* len */
        *buf++ = 0x00; /* Microsoft OUI 00:50:F2 */
        *buf++ = 0x50;
        *buf++ = 0xf2;
        *buf++ = 2; /* WME */
        *buf++ = 0; /* WME info */
        *buf++ = 1; /* WME ver */
        *buf++ = qosinfo; /* U-APSD no in use */

        return buf;
}

void ieee80211_txq_get_depth(struct ieee80211_txq *txq,
                             unsigned long *frame_cnt,
                             unsigned long *byte_cnt)
{
        struct txq_info *txqi = to_txq_info(txq);
        u32 frag_cnt = 0, frag_bytes = 0;
        struct sk_buff *skb;

        skb_queue_walk(&txqi->frags, skb) {
                frag_cnt++;
                frag_bytes += skb->len;
        }

        if (frame_cnt)
                *frame_cnt = txqi->tin.backlog_packets + frag_cnt;

        if (byte_cnt)
                *byte_cnt = txqi->tin.backlog_bytes + frag_bytes;
}
EXPORT_SYMBOL(ieee80211_txq_get_depth);

const u8 ieee80211_ac_to_qos_mask[IEEE80211_NUM_ACS] = {
        IEEE80211_WMM_IE_STA_QOSINFO_AC_VO,
        IEEE80211_WMM_IE_STA_QOSINFO_AC_VI,
        IEEE80211_WMM_IE_STA_QOSINFO_AC_BE,
        IEEE80211_WMM_IE_STA_QOSINFO_AC_BK
};

u16 ieee80211_encode_usf(int listen_interval)
{
        static const int listen_int_usf[] = { 1, 10, 1000, 10000 };
        u16 ui, usf = 0;

        /* find greatest USF */
        while (usf < IEEE80211_MAX_USF) {
                if (listen_interval % listen_int_usf[usf + 1])
                        break;
                usf += 1;
        }
        ui = listen_interval / listen_int_usf[usf];

        /* error if there is a remainder. Should've been checked by user */
        WARN_ON_ONCE(ui > IEEE80211_MAX_UI);
        listen_interval = FIELD_PREP(LISTEN_INT_USF, usf) |
                          FIELD_PREP(LISTEN_INT_UI, ui);

        return (u16) listen_interval;
}

/* this may return more than ieee80211_put_eht_cap() will need */
u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata)
{
        const struct ieee80211_sta_he_cap *he_cap;
        const struct ieee80211_sta_eht_cap *eht_cap;
        struct ieee80211_supported_band *sband;
        bool is_ap;
        u8 n;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return 0;

        he_cap = ieee80211_get_he_iftype_cap_vif(sband, &sdata->vif);
        eht_cap = ieee80211_get_eht_iftype_cap_vif(sband, &sdata->vif);
        if (!he_cap || !eht_cap)
                return 0;

        is_ap = sdata->vif.type == NL80211_IFTYPE_AP;

        n = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem,
                                       &eht_cap->eht_cap_elem,
                                       is_ap);
        return 2 + 1 +
               sizeof(eht_cap->eht_cap_elem) + n +
               ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0],
                                      eht_cap->eht_cap_elem.phy_cap_info);
        return 0;
}

int ieee80211_put_eht_cap(struct sk_buff *skb,
                          struct ieee80211_sub_if_data *sdata,
                          const struct ieee80211_supported_band *sband,
                          const struct ieee80211_conn_settings *conn)
{
        const struct ieee80211_sta_he_cap *he_cap =
                ieee80211_get_he_iftype_cap_vif(sband, &sdata->vif);
        const struct ieee80211_sta_eht_cap *eht_cap =
                ieee80211_get_eht_iftype_cap_vif(sband, &sdata->vif);
        bool for_ap = sdata->vif.type == NL80211_IFTYPE_AP;
        struct ieee80211_eht_cap_elem_fixed fixed;
        struct ieee80211_he_cap_elem he;
        u8 mcs_nss_len, ppet_len;
        u8 orig_mcs_nss_len;
        u8 ie_len;

        if (!conn)
                conn = &ieee80211_conn_settings_unlimited;

        /* Make sure we have place for the IE */
        if (!he_cap || !eht_cap)
                return 0;

        orig_mcs_nss_len = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem,
                                                      &eht_cap->eht_cap_elem,
                                                      for_ap);

        ieee80211_get_adjusted_he_cap(conn, he_cap, &he);

        fixed = eht_cap->eht_cap_elem;

        if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_80)
                fixed.phy_cap_info[6] &=
                        ~IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ;

        if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_160) {
                fixed.phy_cap_info[1] &=
                        ~IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK;
                fixed.phy_cap_info[2] &=
                        ~IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK;
                fixed.phy_cap_info[6] &=
                        ~IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ;
        }

        if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_320) {
                fixed.phy_cap_info[0] &=
                        ~IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ;
                fixed.phy_cap_info[1] &=
                        ~IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK;
                fixed.phy_cap_info[2] &=
                        ~IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK;
                fixed.phy_cap_info[6] &=
                        ~IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ;
        }

        if (conn->bw_limit == IEEE80211_CONN_BW_LIMIT_20)
                fixed.phy_cap_info[0] &=
                        ~IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ;

        mcs_nss_len = ieee80211_eht_mcs_nss_size(&he, &fixed, for_ap);
        ppet_len = ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0],
                                          fixed.phy_cap_info);

        ie_len = 2 + 1 + sizeof(eht_cap->eht_cap_elem) + mcs_nss_len + ppet_len;
        if (skb_tailroom(skb) < ie_len)
                return -ENOBUFS;

        skb_put_u8(skb, WLAN_EID_EXTENSION);
        skb_put_u8(skb, ie_len - 2);
        skb_put_u8(skb, WLAN_EID_EXT_EHT_CAPABILITY);
        skb_put_data(skb, &fixed, sizeof(fixed));

        if (mcs_nss_len == 4 && orig_mcs_nss_len != 4) {
                /*
                 * If the (non-AP) STA became 20 MHz only, then convert from
                 * <=80 to 20-MHz-only format, where MCSes are indicated in
                 * the groups 0-7, 8-9, 10-11, 12-13 rather than just 0-9,
                 * 10-11, 12-13. Thus, use 0-9 for 0-7 and 8-9.
                 */
                skb_put_u8(skb, eht_cap->eht_mcs_nss_supp.bw._80.rx_tx_mcs9_max_nss);
                skb_put_u8(skb, eht_cap->eht_mcs_nss_supp.bw._80.rx_tx_mcs9_max_nss);
                skb_put_u8(skb, eht_cap->eht_mcs_nss_supp.bw._80.rx_tx_mcs11_max_nss);
                skb_put_u8(skb, eht_cap->eht_mcs_nss_supp.bw._80.rx_tx_mcs13_max_nss);
        } else {
                skb_put_data(skb, &eht_cap->eht_mcs_nss_supp, mcs_nss_len);
        }

        if (ppet_len)
                skb_put_data(skb, &eht_cap->eht_ppe_thres, ppet_len);

        return 0;
}

const char *ieee80211_conn_mode_str(enum ieee80211_conn_mode mode)
{
        static const char * const modes[] = {
                [IEEE80211_CONN_MODE_S1G] = "S1G",
                [IEEE80211_CONN_MODE_LEGACY] = "legacy",
                [IEEE80211_CONN_MODE_HT] = "HT",
                [IEEE80211_CONN_MODE_VHT] = "VHT",
                [IEEE80211_CONN_MODE_HE] = "HE",
                [IEEE80211_CONN_MODE_EHT] = "EHT",
        };

        if (WARN_ON(mode >= ARRAY_SIZE(modes)))
                return "<out of range>";

        return modes[mode] ?: "<missing string>";
}

enum ieee80211_conn_bw_limit
ieee80211_min_bw_limit_from_chandef(struct cfg80211_chan_def *chandef)
{
        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_20_NOHT:
        case NL80211_CHAN_WIDTH_20:
                return IEEE80211_CONN_BW_LIMIT_20;
        case NL80211_CHAN_WIDTH_40:
                return IEEE80211_CONN_BW_LIMIT_40;
        case NL80211_CHAN_WIDTH_80:
                return IEEE80211_CONN_BW_LIMIT_80;
        case NL80211_CHAN_WIDTH_80P80:
        case NL80211_CHAN_WIDTH_160:
                return IEEE80211_CONN_BW_LIMIT_160;
        case NL80211_CHAN_WIDTH_320:
                return IEEE80211_CONN_BW_LIMIT_320;
        default:
                WARN(1, "unhandled chandef width %d\n", chandef->width);
                return IEEE80211_CONN_BW_LIMIT_20;
        }
}

void ieee80211_clear_tpe(struct ieee80211_parsed_tpe *tpe)
{
        for (int i = 0; i < 2; i++) {
                tpe->max_local[i].valid = false;
                memset(tpe->max_local[i].power,
                       IEEE80211_TPE_MAX_TX_PWR_NO_CONSTRAINT,
                       sizeof(tpe->max_local[i].power));

                tpe->max_reg_client[i].valid = false;
                memset(tpe->max_reg_client[i].power,
                       IEEE80211_TPE_MAX_TX_PWR_NO_CONSTRAINT,
                       sizeof(tpe->max_reg_client[i].power));

                tpe->psd_local[i].valid = false;
                memset(tpe->psd_local[i].power,
                       IEEE80211_TPE_PSD_NO_LIMIT,
                       sizeof(tpe->psd_local[i].power));

                tpe->psd_reg_client[i].valid = false;
                memset(tpe->psd_reg_client[i].power,
                       IEEE80211_TPE_PSD_NO_LIMIT,
                       sizeof(tpe->psd_reg_client[i].power));
        }
}
































































































    8 







    8 
    1 

































    8 



























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __FIRMWARE_LOADER_H
#define __FIRMWARE_LOADER_H

#include <linux/bitops.h>
#include <linux/firmware.h>
#include <linux/types.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/completion.h>

/**
 * enum fw_opt - options to control firmware loading behaviour
 *
 * @FW_OPT_UEVENT: Enables the fallback mechanism to send a kobject uevent
 *        when the firmware is not found. Userspace is in charge to load the
 *        firmware using the sysfs loading facility.
 * @FW_OPT_NOWAIT: Used to describe the firmware request is asynchronous.
 * @FW_OPT_USERHELPER: Enable the fallback mechanism, in case the direct
 *        filesystem lookup fails at finding the firmware.  For details refer to
 *        firmware_fallback_sysfs().
 * @FW_OPT_NO_WARN: Quiet, avoid printing warning messages.
 * @FW_OPT_NOCACHE: Disables firmware caching. Firmware caching is used to
 *        cache the firmware upon suspend, so that upon resume races against the
 *        firmware file lookup on storage is avoided. Used for calls where the
 *        file may be too big, or where the driver takes charge of its own
 *        firmware caching mechanism.
 * @FW_OPT_NOFALLBACK_SYSFS: Disable the sysfs fallback mechanism. Takes
 *        precedence over &FW_OPT_UEVENT and &FW_OPT_USERHELPER.
 * @FW_OPT_FALLBACK_PLATFORM: Enable fallback to device fw copy embedded in
 *        the platform's main firmware. If both this fallback and the sysfs
 *      fallback are enabled, then this fallback will be tried first.
 * @FW_OPT_PARTIAL: Allow partial read of firmware instead of needing to read
 *        entire file.
 */
enum fw_opt {
        FW_OPT_UEVENT                        = BIT(0),
        FW_OPT_NOWAIT                        = BIT(1),
        FW_OPT_USERHELPER                = BIT(2),
        FW_OPT_NO_WARN                        = BIT(3),
        FW_OPT_NOCACHE                        = BIT(4),
        FW_OPT_NOFALLBACK_SYSFS                = BIT(5),
        FW_OPT_FALLBACK_PLATFORM        = BIT(6),
        FW_OPT_PARTIAL                        = BIT(7),
};

enum fw_status {
        FW_STATUS_UNKNOWN,
        FW_STATUS_LOADING,
        FW_STATUS_DONE,
        FW_STATUS_ABORTED,
};

/*
 * Concurrent request_firmware() for the same firmware need to be
 * serialized.  struct fw_state is simple state machine which hold the
 * state of the firmware loading.
 */
struct fw_state {
        struct completion completion;
        enum fw_status status;
};

struct fw_priv {
        struct kref ref;
        struct list_head list;
        struct firmware_cache *fwc;
        struct fw_state fw_st;
        void *data;
        size_t size;
        size_t allocated_size;
        size_t offset;
        u32 opt_flags;
#ifdef CONFIG_FW_LOADER_PAGED_BUF
        bool is_paged_buf;
        struct page **pages;
        int nr_pages;
        int page_array_size;
#endif
#ifdef CONFIG_FW_LOADER_USER_HELPER
        bool need_uevent;
        struct list_head pending_list;
#endif
        const char *fw_name;
};

extern struct mutex fw_lock;
extern struct firmware_cache fw_cache;
extern bool fw_load_abort_all;

static inline bool __fw_state_check(struct fw_priv *fw_priv,
                                    enum fw_status status)
{
        struct fw_state *fw_st = &fw_priv->fw_st;

        return fw_st->status == status;
}

static inline int __fw_state_wait_common(struct fw_priv *fw_priv, long timeout)
{
        struct fw_state *fw_st = &fw_priv->fw_st;
        long ret;

        ret = wait_for_completion_killable_timeout(&fw_st->completion, timeout);
        if (ret != 0 && fw_st->status == FW_STATUS_ABORTED)
                return -ENOENT;
        if (!ret)
                return -ETIMEDOUT;

        return ret < 0 ? ret : 0;
}

static inline void __fw_state_set(struct fw_priv *fw_priv,
                                  enum fw_status status)
{
        struct fw_state *fw_st = &fw_priv->fw_st;

        WRITE_ONCE(fw_st->status, status);

        if (status == FW_STATUS_DONE || status == FW_STATUS_ABORTED) {
#ifdef CONFIG_FW_LOADER_USER_HELPER
                /*
                 * Doing this here ensures that the fw_priv is deleted from
                 * the pending list in all abort/done paths.
                 */
                list_del_init(&fw_priv->pending_list);
#endif
                complete_all(&fw_st->completion);
        }
}

static inline void fw_state_aborted(struct fw_priv *fw_priv)
{
        __fw_state_set(fw_priv, FW_STATUS_ABORTED);
}

static inline bool fw_state_is_aborted(struct fw_priv *fw_priv)
{
        return __fw_state_check(fw_priv, FW_STATUS_ABORTED);
}

static inline void fw_state_start(struct fw_priv *fw_priv)
{
        __fw_state_set(fw_priv, FW_STATUS_LOADING);
}

static inline void fw_state_done(struct fw_priv *fw_priv)
{
        __fw_state_set(fw_priv, FW_STATUS_DONE);
}

static inline bool fw_state_is_done(struct fw_priv *fw_priv)
{
        return __fw_state_check(fw_priv, FW_STATUS_DONE);
}

static inline bool fw_state_is_loading(struct fw_priv *fw_priv)
{
        return __fw_state_check(fw_priv, FW_STATUS_LOADING);
}

int alloc_lookup_fw_priv(const char *fw_name, struct firmware_cache *fwc,
                         struct fw_priv **fw_priv, void *dbuf, size_t size,
                         size_t offset, u32 opt_flags);
int assign_fw(struct firmware *fw, struct device *device);
void free_fw_priv(struct fw_priv *fw_priv);
void fw_state_init(struct fw_priv *fw_priv);

#ifdef CONFIG_FW_LOADER
bool firmware_is_builtin(const struct firmware *fw);
bool firmware_request_builtin_buf(struct firmware *fw, const char *name,
                                  void *buf, size_t size);
#else /* module case */
static inline bool firmware_is_builtin(const struct firmware *fw)
{
        return false;
}
static inline bool firmware_request_builtin_buf(struct firmware *fw,
                                                const char *name,
                                                void *buf, size_t size)
{
        return false;
}
#endif

#ifdef CONFIG_FW_LOADER_PAGED_BUF
void fw_free_paged_buf(struct fw_priv *fw_priv);
int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed);
int fw_map_paged_buf(struct fw_priv *fw_priv);
bool fw_is_paged_buf(struct fw_priv *fw_priv);
#else
static inline void fw_free_paged_buf(struct fw_priv *fw_priv) {}
static inline int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed) { return -ENXIO; }
static inline int fw_map_paged_buf(struct fw_priv *fw_priv) { return -ENXIO; }
static inline bool fw_is_paged_buf(struct fw_priv *fw_priv) { return false; }
#endif

#endif /* __FIRMWARE_LOADER_H */











































   84 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _TRACE_SYSCALL_H
#define _TRACE_SYSCALL_H

#include <linux/tracepoint.h>
#include <linux/unistd.h>
#include <linux/trace_events.h>
#include <linux/thread_info.h>

#include <asm/ptrace.h>


/*
 * A syscall entry in the ftrace syscalls array.
 *
 * @name: name of the syscall
 * @syscall_nr: number of the syscall
 * @nb_args: number of parameters it takes
 * @types: list of types as strings
 * @args: list of args as strings (args[i] matches types[i])
 * @enter_fields: list of fields for syscall_enter trace event
 * @enter_event: associated syscall_enter trace event
 * @exit_event: associated syscall_exit trace event
 */
struct syscall_metadata {
        const char        *name;
        int                syscall_nr;
        int                nb_args;
        const char        **types;
        const char        **args;
        struct list_head enter_fields;

        struct trace_event_call *enter_event;
        struct trace_event_call *exit_event;
};

#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_HAVE_SYSCALL_TRACEPOINTS)
static inline void syscall_tracepoint_update(struct task_struct *p)
{
        if (test_syscall_work(SYSCALL_TRACEPOINT))
                set_task_syscall_work(p, SYSCALL_TRACEPOINT);
        else
                clear_task_syscall_work(p, SYSCALL_TRACEPOINT);
}
#else
static inline void syscall_tracepoint_update(struct task_struct *p)
{
}
#endif

#endif /* _TRACE_SYSCALL_H */

















































 1824 


 2390 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2005-2010 IBM Corporation
 *
 * Authors:
 * Mimi Zohar <zohar@us.ibm.com>
 * Kylene Hall <kjhall@us.ibm.com>
 *
 * File: evm.h
 */

#ifndef __INTEGRITY_EVM_H
#define __INTEGRITY_EVM_H

#include <linux/xattr.h>
#include <linux/security.h>

#include "../integrity.h"

#define EVM_INIT_HMAC        0x0001
#define EVM_INIT_X509        0x0002
#define EVM_ALLOW_METADATA_WRITES        0x0004
#define EVM_SETUP_COMPLETE 0x80000000 /* userland has signaled key load */

#define EVM_KEY_MASK (EVM_INIT_HMAC | EVM_INIT_X509)
#define EVM_INIT_MASK (EVM_INIT_HMAC | EVM_INIT_X509 | EVM_SETUP_COMPLETE | \
                       EVM_ALLOW_METADATA_WRITES)

struct xattr_list {
        struct list_head list;
        char *name;
        bool enabled;
};

#define EVM_NEW_FILE                        0x00000001
#define EVM_IMMUTABLE_DIGSIG                0x00000002

/* EVM integrity metadata associated with an inode */
struct evm_iint_cache {
        unsigned long flags;
        enum integrity_status evm_status:4;
        struct integrity_inode_attributes metadata_inode;
};

extern struct lsm_blob_sizes evm_blob_sizes;

static inline struct evm_iint_cache *evm_iint_inode(const struct inode *inode)
{
        if (unlikely(!inode->i_security))
                return NULL;

        return inode->i_security + evm_blob_sizes.lbs_inode;
}

extern int evm_initialized;

#define EVM_ATTR_FSUUID                0x0001

extern int evm_hmac_attrs;

/* List of EVM protected security xattrs */
extern struct list_head evm_config_xattrnames;

struct evm_digest {
        struct ima_digest_data_hdr hdr;
        char digest[IMA_MAX_DIGEST_SIZE];
} __packed;

int evm_protected_xattr(const char *req_xattr_name);

int evm_init_key(void);
int evm_update_evmxattr(struct dentry *dentry,
                        const char *req_xattr_name,
                        const char *req_xattr_value,
                        size_t req_xattr_value_len);
int evm_calc_hmac(struct dentry *dentry, const char *req_xattr_name,
                  const char *req_xattr_value,
                  size_t req_xattr_value_len, struct evm_digest *data,
                  struct evm_iint_cache *iint);
int evm_calc_hash(struct dentry *dentry, const char *req_xattr_name,
                  const char *req_xattr_value,
                  size_t req_xattr_value_len, char type,
                  struct evm_digest *data, struct evm_iint_cache *iint);
int evm_init_hmac(struct inode *inode, const struct xattr *xattrs,
                  char *hmac_val);
int evm_init_secfs(void);

#endif


























































































































































 6884 



 6869 

 6933 








 6864 







 6881 
 6863 


 6934 
 6886 
 6927 

 6869 

















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
// SPDX-License-Identifier: GPL-2.0-only
/*
 * AppArmor security module
 *
 * This file contains AppArmor network mediation
 *
 * Copyright (C) 1998-2008 Novell/SUSE
 * Copyright 2009-2017 Canonical Ltd.
 */

#include "include/apparmor.h"
#include "include/audit.h"
#include "include/cred.h"
#include "include/label.h"
#include "include/net.h"
#include "include/policy.h"
#include "include/secid.h"

#include "net_names.h"


struct aa_sfs_entry aa_sfs_entry_network[] = {
        AA_SFS_FILE_STRING("af_mask",        AA_SFS_AF_MASK),
        { }
};

static const char * const net_mask_names[] = {
        "unknown",
        "send",
        "receive",
        "unknown",

        "create",
        "shutdown",
        "connect",
        "unknown",

        "setattr",
        "getattr",
        "setcred",
        "getcred",

        "chmod",
        "chown",
        "chgrp",
        "lock",

        "mmap",
        "mprot",
        "unknown",
        "unknown",

        "accept",
        "bind",
        "listen",
        "unknown",

        "setopt",
        "getopt",
        "unknown",
        "unknown",

        "unknown",
        "unknown",
        "unknown",
        "unknown",
};


/* audit callback for net specific fields */
void audit_net_cb(struct audit_buffer *ab, void *va)
{
        struct common_audit_data *sa = va;
        struct apparmor_audit_data *ad = aad(sa);

        if (address_family_names[sa->u.net->family])
                audit_log_format(ab, " family=\"%s\"",
                                 address_family_names[sa->u.net->family]);
        else
                audit_log_format(ab, " family=\"unknown(%d)\"",
                                 sa->u.net->family);
        if (sock_type_names[ad->net.type])
                audit_log_format(ab, " sock_type=\"%s\"",
                                 sock_type_names[ad->net.type]);
        else
                audit_log_format(ab, " sock_type=\"unknown(%d)\"",
                                 ad->net.type);
        audit_log_format(ab, " protocol=%d", ad->net.protocol);

        if (ad->request & NET_PERMS_MASK) {
                audit_log_format(ab, " requested_mask=");
                aa_audit_perm_mask(ab, ad->request, NULL, 0,
                                   net_mask_names, NET_PERMS_MASK);

                if (ad->denied & NET_PERMS_MASK) {
                        audit_log_format(ab, " denied_mask=");
                        aa_audit_perm_mask(ab, ad->denied, NULL, 0,
                                           net_mask_names, NET_PERMS_MASK);
                }
        }
        if (ad->peer) {
                audit_log_format(ab, " peer=");
                aa_label_xaudit(ab, labels_ns(ad->subj_label), ad->peer,
                                FLAGS_NONE, GFP_ATOMIC);
        }
}

/* Generic af perm */
int aa_profile_af_perm(struct aa_profile *profile,
                       struct apparmor_audit_data *ad, u32 request, u16 family,
                       int type)
{
        struct aa_ruleset *rules = list_first_entry(&profile->rules,
                                                    typeof(*rules), list);
        struct aa_perms perms = { };
        aa_state_t state;
        __be16 buffer[2];

        AA_BUG(family >= AF_MAX);
        AA_BUG(type < 0 || type >= SOCK_MAX);

        if (profile_unconfined(profile))
                return 0;
        state = RULE_MEDIATES(rules, AA_CLASS_NET);
        if (!state)
                return 0;

        buffer[0] = cpu_to_be16(family);
        buffer[1] = cpu_to_be16((u16) type);
        state = aa_dfa_match_len(rules->policy->dfa, state, (char *) &buffer,
                                 4);
        perms = *aa_lookup_perms(rules->policy, state);
        aa_apply_modes_to_perms(profile, &perms);

        return aa_check_perms(profile, &perms, request, ad, audit_net_cb);
}

int aa_af_perm(const struct cred *subj_cred, struct aa_label *label,
               const char *op, u32 request, u16 family, int type, int protocol)
{
        struct aa_profile *profile;
        DEFINE_AUDIT_NET(ad, op, NULL, family, type, protocol);

        return fn_for_each_confined(label, profile,
                        aa_profile_af_perm(profile, &ad, request, family,
                                           type));
}

static int aa_label_sk_perm(const struct cred *subj_cred,
                            struct aa_label *label,
                            const char *op, u32 request,
                            struct sock *sk)
{
        struct aa_sk_ctx *ctx = aa_sock(sk);
        int error = 0;

        AA_BUG(!label);
        AA_BUG(!sk);

        if (ctx->label != kernel_t && !unconfined(label)) {
                struct aa_profile *profile;
                DEFINE_AUDIT_SK(ad, op, sk);

                ad.subj_cred = subj_cred;
                error = fn_for_each_confined(label, profile,
                            aa_profile_af_sk_perm(profile, &ad, request, sk));
        }

        return error;
}

int aa_sk_perm(const char *op, u32 request, struct sock *sk)
{
        struct aa_label *label;
        int error;

        AA_BUG(!sk);
        AA_BUG(in_interrupt());

        /* TODO: switch to begin_current_label ???? */
        label = begin_current_label_crit_section();
        error = aa_label_sk_perm(current_cred(), label, op, request, sk);
        end_current_label_crit_section(label);

        return error;
}


int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label,
                      const char *op, u32 request, struct socket *sock)
{
        AA_BUG(!label);
        AA_BUG(!sock);
        AA_BUG(!sock->sk);

        return aa_label_sk_perm(subj_cred, label, op, request, sock->sk);
}

#ifdef CONFIG_NETWORK_SECMARK
static int apparmor_secmark_init(struct aa_secmark *secmark)
{
        struct aa_label *label;

        if (secmark->label[0] == '*') {
                secmark->secid = AA_SECID_WILDCARD;
                return 0;
        }

        label = aa_label_strn_parse(&root_ns->unconfined->label,
                                    secmark->label, strlen(secmark->label),
                                    GFP_ATOMIC, false, false);

        if (IS_ERR(label))
                return PTR_ERR(label);

        secmark->secid = label->secid;

        return 0;
}

static int aa_secmark_perm(struct aa_profile *profile, u32 request, u32 secid,
                           struct apparmor_audit_data *ad)
{
        int i, ret;
        struct aa_perms perms = { };
        struct aa_ruleset *rules = list_first_entry(&profile->rules,
                                                    typeof(*rules), list);

        if (rules->secmark_count == 0)
                return 0;

        for (i = 0; i < rules->secmark_count; i++) {
                if (!rules->secmark[i].secid) {
                        ret = apparmor_secmark_init(&rules->secmark[i]);
                        if (ret)
                                return ret;
                }

                if (rules->secmark[i].secid == secid ||
                    rules->secmark[i].secid == AA_SECID_WILDCARD) {
                        if (rules->secmark[i].deny)
                                perms.deny = ALL_PERMS_MASK;
                        else
                                perms.allow = ALL_PERMS_MASK;

                        if (rules->secmark[i].audit)
                                perms.audit = ALL_PERMS_MASK;
                }
        }

        aa_apply_modes_to_perms(profile, &perms);

        return aa_check_perms(profile, &perms, request, ad, audit_net_cb);
}

int apparmor_secmark_check(struct aa_label *label, char *op, u32 request,
                           u32 secid, const struct sock *sk)
{
        struct aa_profile *profile;
        DEFINE_AUDIT_SK(ad, op, sk);

        return fn_for_each_confined(label, profile,
                                    aa_secmark_perm(profile, request, secid,
                                                    &ad));
}
#endif





















































































































































































































































































































































































































































































































































































































































































































































































    1 



    1 






    1 




    1 
    1 















    2 



    2 




    2 





    2 







































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2017-2018 HUAWEI, Inc.
 *             https://www.huawei.com/
 * Copyright (C) 2021, Alibaba Cloud
 */
#include <linux/statfs.h>
#include <linux/seq_file.h>
#include <linux/crc32c.h>
#include <linux/fs_context.h>
#include <linux/fs_parser.h>
#include <linux/exportfs.h>
#include <linux/backing-dev.h>
#include "xattr.h"

#define CREATE_TRACE_POINTS
#include <trace/events/erofs.h>

static struct kmem_cache *erofs_inode_cachep __read_mostly;

void _erofs_printk(struct super_block *sb, const char *fmt, ...)
{
        struct va_format vaf;
        va_list args;
        int level;

        va_start(args, fmt);

        level = printk_get_level(fmt);
        vaf.fmt = printk_skip_level(fmt);
        vaf.va = &args;
        if (sb)
                printk("%c%cerofs (device %s): %pV",
                                KERN_SOH_ASCII, level, sb->s_id, &vaf);
        else
                printk("%c%cerofs: %pV", KERN_SOH_ASCII, level, &vaf);
        va_end(args);
}

static int erofs_superblock_csum_verify(struct super_block *sb, void *sbdata)
{
        size_t len = 1 << EROFS_SB(sb)->blkszbits;
        struct erofs_super_block *dsb;
        u32 expected_crc, crc;

        if (len > EROFS_SUPER_OFFSET)
                len -= EROFS_SUPER_OFFSET;

        dsb = kmemdup(sbdata + EROFS_SUPER_OFFSET, len, GFP_KERNEL);
        if (!dsb)
                return -ENOMEM;

        expected_crc = le32_to_cpu(dsb->checksum);
        dsb->checksum = 0;
        /* to allow for x86 boot sectors and other oddities. */
        crc = crc32c(~0, dsb, len);
        kfree(dsb);

        if (crc != expected_crc) {
                erofs_err(sb, "invalid checksum 0x%08x, 0x%08x expected",
                          crc, expected_crc);
                return -EBADMSG;
        }
        return 0;
}

static void erofs_inode_init_once(void *ptr)
{
        struct erofs_inode *vi = ptr;

        inode_init_once(&vi->vfs_inode);
}

static struct inode *erofs_alloc_inode(struct super_block *sb)
{
        struct erofs_inode *vi =
                alloc_inode_sb(sb, erofs_inode_cachep, GFP_KERNEL);

        if (!vi)
                return NULL;

        /* zero out everything except vfs_inode */
        memset(vi, 0, offsetof(struct erofs_inode, vfs_inode));
        return &vi->vfs_inode;
}

static void erofs_free_inode(struct inode *inode)
{
        struct erofs_inode *vi = EROFS_I(inode);

        if (inode->i_op == &erofs_fast_symlink_iops)
                kfree(inode->i_link);
        kfree(vi->xattr_shared_xattrs);
        kmem_cache_free(erofs_inode_cachep, vi);
}

/* read variable-sized metadata, offset will be aligned by 4-byte */
void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf,
                          erofs_off_t *offset, int *lengthp)
{
        u8 *buffer, *ptr;
        int len, i, cnt;

        *offset = round_up(*offset, 4);
        ptr = erofs_bread(buf, *offset, EROFS_KMAP);
        if (IS_ERR(ptr))
                return ptr;

        len = le16_to_cpu(*(__le16 *)ptr);
        if (!len)
                len = U16_MAX + 1;
        buffer = kmalloc(len, GFP_KERNEL);
        if (!buffer)
                return ERR_PTR(-ENOMEM);
        *offset += sizeof(__le16);
        *lengthp = len;

        for (i = 0; i < len; i += cnt) {
                cnt = min_t(int, sb->s_blocksize - erofs_blkoff(sb, *offset),
                            len - i);
                ptr = erofs_bread(buf, *offset, EROFS_KMAP);
                if (IS_ERR(ptr)) {
                        kfree(buffer);
                        return ptr;
                }
                memcpy(buffer + i, ptr, cnt);
                *offset += cnt;
        }
        return buffer;
}

#ifndef CONFIG_EROFS_FS_ZIP
static int z_erofs_parse_cfgs(struct super_block *sb,
                              struct erofs_super_block *dsb)
{
        if (!dsb->u1.available_compr_algs)
                return 0;

        erofs_err(sb, "compression disabled, unable to mount compressed EROFS");
        return -EOPNOTSUPP;
}
#endif

static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb,
                             struct erofs_device_info *dif, erofs_off_t *pos)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        struct erofs_fscache *fscache;
        struct erofs_deviceslot *dis;
        struct file *file;

        dis = erofs_read_metabuf(buf, sb, *pos, EROFS_KMAP);
        if (IS_ERR(dis))
                return PTR_ERR(dis);

        if (!sbi->devs->flatdev && !dif->path) {
                if (!dis->tag[0]) {
                        erofs_err(sb, "empty device tag @ pos %llu", *pos);
                        return -EINVAL;
                }
                dif->path = kmemdup_nul(dis->tag, sizeof(dis->tag), GFP_KERNEL);
                if (!dif->path)
                        return -ENOMEM;
        }

        if (erofs_is_fscache_mode(sb)) {
                fscache = erofs_fscache_register_cookie(sb, dif->path, 0);
                if (IS_ERR(fscache))
                        return PTR_ERR(fscache);
                dif->fscache = fscache;
        } else if (!sbi->devs->flatdev) {
                file = erofs_is_fileio_mode(sbi) ?
                                filp_open(dif->path, O_RDONLY | O_LARGEFILE, 0) :
                                bdev_file_open_by_path(dif->path,
                                                BLK_OPEN_READ, sb->s_type, NULL);
                if (IS_ERR(file))
                        return PTR_ERR(file);

                if (!erofs_is_fileio_mode(sbi)) {
                        dif->dax_dev = fs_dax_get_by_bdev(file_bdev(file),
                                        &dif->dax_part_off, NULL, NULL);
                } else if (!S_ISREG(file_inode(file)->i_mode)) {
                        fput(file);
                        return -EINVAL;
                }
                dif->file = file;
        }

        dif->blocks = le32_to_cpu(dis->blocks);
        dif->mapped_blkaddr = le32_to_cpu(dis->mapped_blkaddr);
        sbi->total_blocks += dif->blocks;
        *pos += EROFS_DEVT_SLOT_SIZE;
        return 0;
}

static int erofs_scan_devices(struct super_block *sb,
                              struct erofs_super_block *dsb)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        unsigned int ondisk_extradevs;
        erofs_off_t pos;
        struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
        struct erofs_device_info *dif;
        int id, err = 0;

        sbi->total_blocks = sbi->primarydevice_blocks;
        if (!erofs_sb_has_device_table(sbi))
                ondisk_extradevs = 0;
        else
                ondisk_extradevs = le16_to_cpu(dsb->extra_devices);

        if (sbi->devs->extra_devices &&
            ondisk_extradevs != sbi->devs->extra_devices) {
                erofs_err(sb, "extra devices don't match (ondisk %u, given %u)",
                          ondisk_extradevs, sbi->devs->extra_devices);
                return -EINVAL;
        }
        if (!ondisk_extradevs)
                return 0;

        if (!sbi->devs->extra_devices && !erofs_is_fscache_mode(sb))
                sbi->devs->flatdev = true;

        sbi->device_id_mask = roundup_pow_of_two(ondisk_extradevs + 1) - 1;
        pos = le16_to_cpu(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE;
        down_read(&sbi->devs->rwsem);
        if (sbi->devs->extra_devices) {
                idr_for_each_entry(&sbi->devs->tree, dif, id) {
                        err = erofs_init_device(&buf, sb, dif, &pos);
                        if (err)
                                break;
                }
        } else {
                for (id = 0; id < ondisk_extradevs; id++) {
                        dif = kzalloc(sizeof(*dif), GFP_KERNEL);
                        if (!dif) {
                                err = -ENOMEM;
                                break;
                        }

                        err = idr_alloc(&sbi->devs->tree, dif, 0, 0, GFP_KERNEL);
                        if (err < 0) {
                                kfree(dif);
                                break;
                        }
                        ++sbi->devs->extra_devices;

                        err = erofs_init_device(&buf, sb, dif, &pos);
                        if (err)
                                break;
                }
        }
        up_read(&sbi->devs->rwsem);
        erofs_put_metabuf(&buf);
        return err;
}

static int erofs_read_superblock(struct super_block *sb)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
        struct erofs_super_block *dsb;
        void *data;
        int ret;

        data = erofs_read_metabuf(&buf, sb, 0, EROFS_KMAP);
        if (IS_ERR(data)) {
                erofs_err(sb, "cannot read erofs superblock");
                return PTR_ERR(data);
        }

        dsb = (struct erofs_super_block *)(data + EROFS_SUPER_OFFSET);
        ret = -EINVAL;
        if (le32_to_cpu(dsb->magic) != EROFS_SUPER_MAGIC_V1) {
                erofs_err(sb, "cannot find valid erofs superblock");
                goto out;
        }

        sbi->blkszbits  = dsb->blkszbits;
        if (sbi->blkszbits < 9 || sbi->blkszbits > PAGE_SHIFT) {
                erofs_err(sb, "blkszbits %u isn't supported", sbi->blkszbits);
                goto out;
        }
        if (dsb->dirblkbits) {
                erofs_err(sb, "dirblkbits %u isn't supported", dsb->dirblkbits);
                goto out;
        }

        sbi->feature_compat = le32_to_cpu(dsb->feature_compat);
        if (erofs_sb_has_sb_chksum(sbi)) {
                ret = erofs_superblock_csum_verify(sb, data);
                if (ret)
                        goto out;
        }

        ret = -EINVAL;
        sbi->feature_incompat = le32_to_cpu(dsb->feature_incompat);
        if (sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT) {
                erofs_err(sb, "unidentified incompatible feature %x, please upgrade kernel",
                          sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT);
                goto out;
        }

        sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE;
        if (sbi->sb_size > PAGE_SIZE - EROFS_SUPER_OFFSET) {
                erofs_err(sb, "invalid sb_extslots %u (more than a fs block)",
                          sbi->sb_size);
                goto out;
        }
        sbi->primarydevice_blocks = le32_to_cpu(dsb->blocks);
        sbi->meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr);
#ifdef CONFIG_EROFS_FS_XATTR
        sbi->xattr_blkaddr = le32_to_cpu(dsb->xattr_blkaddr);
        sbi->xattr_prefix_start = le32_to_cpu(dsb->xattr_prefix_start);
        sbi->xattr_prefix_count = dsb->xattr_prefix_count;
        sbi->xattr_filter_reserved = dsb->xattr_filter_reserved;
#endif
        sbi->islotbits = ilog2(sizeof(struct erofs_inode_compact));
        sbi->root_nid = le16_to_cpu(dsb->root_nid);
        sbi->packed_nid = le64_to_cpu(dsb->packed_nid);
        sbi->inos = le64_to_cpu(dsb->inos);

        sbi->build_time = le64_to_cpu(dsb->build_time);
        sbi->build_time_nsec = le32_to_cpu(dsb->build_time_nsec);

        super_set_uuid(sb, (void *)dsb->uuid, sizeof(dsb->uuid));

        ret = strscpy(sbi->volume_name, dsb->volume_name,
                      sizeof(dsb->volume_name));
        if (ret < 0) {        /* -E2BIG */
                erofs_err(sb, "bad volume name without NIL terminator");
                ret = -EFSCORRUPTED;
                goto out;
        }

        /* parse on-disk compression configurations */
        ret = z_erofs_parse_cfgs(sb, dsb);
        if (ret < 0)
                goto out;

        /* handle multiple devices */
        ret = erofs_scan_devices(sb, dsb);

        if (erofs_is_fscache_mode(sb))
                erofs_info(sb, "[deprecated] fscache-based on-demand read feature in use. Use at your own risk!");
out:
        erofs_put_metabuf(&buf);
        return ret;
}

static void erofs_default_options(struct erofs_sb_info *sbi)
{
#ifdef CONFIG_EROFS_FS_ZIP
        sbi->opt.cache_strategy = EROFS_ZIP_CACHE_READAROUND;
        sbi->opt.max_sync_decompress_pages = 3;
        sbi->opt.sync_decompress = EROFS_SYNC_DECOMPRESS_AUTO;
#endif
#ifdef CONFIG_EROFS_FS_XATTR
        set_opt(&sbi->opt, XATTR_USER);
#endif
#ifdef CONFIG_EROFS_FS_POSIX_ACL
        set_opt(&sbi->opt, POSIX_ACL);
#endif
}

enum {
        Opt_user_xattr,
        Opt_acl,
        Opt_cache_strategy,
        Opt_dax,
        Opt_dax_enum,
        Opt_device,
        Opt_fsid,
        Opt_domain_id,
        Opt_err
};

static const struct constant_table erofs_param_cache_strategy[] = {
        {"disabled",        EROFS_ZIP_CACHE_DISABLED},
        {"readahead",        EROFS_ZIP_CACHE_READAHEAD},
        {"readaround",        EROFS_ZIP_CACHE_READAROUND},
        {}
};

static const struct constant_table erofs_dax_param_enums[] = {
        {"always",        EROFS_MOUNT_DAX_ALWAYS},
        {"never",        EROFS_MOUNT_DAX_NEVER},
        {}
};

static const struct fs_parameter_spec erofs_fs_parameters[] = {
        fsparam_flag_no("user_xattr",        Opt_user_xattr),
        fsparam_flag_no("acl",                Opt_acl),
        fsparam_enum("cache_strategy",        Opt_cache_strategy,
                     erofs_param_cache_strategy),
        fsparam_flag("dax",             Opt_dax),
        fsparam_enum("dax",                Opt_dax_enum, erofs_dax_param_enums),
        fsparam_string("device",        Opt_device),
        fsparam_string("fsid",                Opt_fsid),
        fsparam_string("domain_id",        Opt_domain_id),
        {}
};

static bool erofs_fc_set_dax_mode(struct fs_context *fc, unsigned int mode)
{
#ifdef CONFIG_FS_DAX
        struct erofs_sb_info *sbi = fc->s_fs_info;

        switch (mode) {
        case EROFS_MOUNT_DAX_ALWAYS:
                set_opt(&sbi->opt, DAX_ALWAYS);
                clear_opt(&sbi->opt, DAX_NEVER);
                return true;
        case EROFS_MOUNT_DAX_NEVER:
                set_opt(&sbi->opt, DAX_NEVER);
                clear_opt(&sbi->opt, DAX_ALWAYS);
                return true;
        default:
                DBG_BUGON(1);
                return false;
        }
#else
        errorfc(fc, "dax options not supported");
        return false;
#endif
}

static int erofs_fc_parse_param(struct fs_context *fc,
                                struct fs_parameter *param)
{
        struct erofs_sb_info *sbi = fc->s_fs_info;
        struct fs_parse_result result;
        struct erofs_device_info *dif;
        int opt, ret;

        opt = fs_parse(fc, erofs_fs_parameters, param, &result);
        if (opt < 0)
                return opt;

        switch (opt) {
        case Opt_user_xattr:
#ifdef CONFIG_EROFS_FS_XATTR
                if (result.boolean)
                        set_opt(&sbi->opt, XATTR_USER);
                else
                        clear_opt(&sbi->opt, XATTR_USER);
#else
                errorfc(fc, "{,no}user_xattr options not supported");
#endif
                break;
        case Opt_acl:
#ifdef CONFIG_EROFS_FS_POSIX_ACL
                if (result.boolean)
                        set_opt(&sbi->opt, POSIX_ACL);
                else
                        clear_opt(&sbi->opt, POSIX_ACL);
#else
                errorfc(fc, "{,no}acl options not supported");
#endif
                break;
        case Opt_cache_strategy:
#ifdef CONFIG_EROFS_FS_ZIP
                sbi->opt.cache_strategy = result.uint_32;
#else
                errorfc(fc, "compression not supported, cache_strategy ignored");
#endif
                break;
        case Opt_dax:
                if (!erofs_fc_set_dax_mode(fc, EROFS_MOUNT_DAX_ALWAYS))
                        return -EINVAL;
                break;
        case Opt_dax_enum:
                if (!erofs_fc_set_dax_mode(fc, result.uint_32))
                        return -EINVAL;
                break;
        case Opt_device:
                dif = kzalloc(sizeof(*dif), GFP_KERNEL);
                if (!dif)
                        return -ENOMEM;
                dif->path = kstrdup(param->string, GFP_KERNEL);
                if (!dif->path) {
                        kfree(dif);
                        return -ENOMEM;
                }
                down_write(&sbi->devs->rwsem);
                ret = idr_alloc(&sbi->devs->tree, dif, 0, 0, GFP_KERNEL);
                up_write(&sbi->devs->rwsem);
                if (ret < 0) {
                        kfree(dif->path);
                        kfree(dif);
                        return ret;
                }
                ++sbi->devs->extra_devices;
                break;
#ifdef CONFIG_EROFS_FS_ONDEMAND
        case Opt_fsid:
                kfree(sbi->fsid);
                sbi->fsid = kstrdup(param->string, GFP_KERNEL);
                if (!sbi->fsid)
                        return -ENOMEM;
                break;
        case Opt_domain_id:
                kfree(sbi->domain_id);
                sbi->domain_id = kstrdup(param->string, GFP_KERNEL);
                if (!sbi->domain_id)
                        return -ENOMEM;
                break;
#else
        case Opt_fsid:
        case Opt_domain_id:
                errorfc(fc, "%s option not supported", erofs_fs_parameters[opt].name);
                break;
#endif
        default:
                return -ENOPARAM;
        }
        return 0;
}

static struct inode *erofs_nfs_get_inode(struct super_block *sb,
                                         u64 ino, u32 generation)
{
        return erofs_iget(sb, ino);
}

static struct dentry *erofs_fh_to_dentry(struct super_block *sb,
                struct fid *fid, int fh_len, int fh_type)
{
        return generic_fh_to_dentry(sb, fid, fh_len, fh_type,
                                    erofs_nfs_get_inode);
}

static struct dentry *erofs_fh_to_parent(struct super_block *sb,
                struct fid *fid, int fh_len, int fh_type)
{
        return generic_fh_to_parent(sb, fid, fh_len, fh_type,
                                    erofs_nfs_get_inode);
}

static struct dentry *erofs_get_parent(struct dentry *child)
{
        erofs_nid_t nid;
        unsigned int d_type;
        int err;

        err = erofs_namei(d_inode(child), &dotdot_name, &nid, &d_type);
        if (err)
                return ERR_PTR(err);
        return d_obtain_alias(erofs_iget(child->d_sb, nid));
}

static const struct export_operations erofs_export_ops = {
        .encode_fh = generic_encode_ino32_fh,
        .fh_to_dentry = erofs_fh_to_dentry,
        .fh_to_parent = erofs_fh_to_parent,
        .get_parent = erofs_get_parent,
};

static void erofs_set_sysfs_name(struct super_block *sb)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);

        if (sbi->domain_id)
                super_set_sysfs_name_generic(sb, "%s,%s", sbi->domain_id,
                                             sbi->fsid);
        else if (sbi->fsid)
                super_set_sysfs_name_generic(sb, "%s", sbi->fsid);
        else if (erofs_is_fileio_mode(sbi))
                super_set_sysfs_name_generic(sb, "%s",
                                             bdi_dev_name(sb->s_bdi));
        else
                super_set_sysfs_name_id(sb);
}

static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
{
        struct inode *inode;
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        int err;

        sb->s_magic = EROFS_SUPER_MAGIC;
        sb->s_flags |= SB_RDONLY | SB_NOATIME;
        sb->s_maxbytes = MAX_LFS_FILESIZE;
        sb->s_op = &erofs_sops;

        sbi->blkszbits = PAGE_SHIFT;
        if (!sb->s_bdev) {
                sb->s_blocksize = PAGE_SIZE;
                sb->s_blocksize_bits = PAGE_SHIFT;

                if (erofs_is_fscache_mode(sb)) {
                        err = erofs_fscache_register_fs(sb);
                        if (err)
                                return err;
                }
                err = super_setup_bdi(sb);
                if (err)
                        return err;
        } else {
                if (!sb_set_blocksize(sb, PAGE_SIZE)) {
                        errorfc(fc, "failed to set initial blksize");
                        return -EINVAL;
                }

                sbi->dax_dev = fs_dax_get_by_bdev(sb->s_bdev,
                                                  &sbi->dax_part_off,
                                                  NULL, NULL);
        }

        err = erofs_read_superblock(sb);
        if (err)
                return err;

        if (sb->s_blocksize_bits != sbi->blkszbits) {
                if (erofs_is_fscache_mode(sb)) {
                        errorfc(fc, "unsupported blksize for fscache mode");
                        return -EINVAL;
                }

                if (erofs_is_fileio_mode(sbi)) {
                        sb->s_blocksize = 1 << sbi->blkszbits;
                        sb->s_blocksize_bits = sbi->blkszbits;
                } else if (!sb_set_blocksize(sb, 1 << sbi->blkszbits)) {
                        errorfc(fc, "failed to set erofs blksize");
                        return -EINVAL;
                }
        }

        if (test_opt(&sbi->opt, DAX_ALWAYS)) {
                if (!sbi->dax_dev) {
                        errorfc(fc, "DAX unsupported by block device. Turning off DAX.");
                        clear_opt(&sbi->opt, DAX_ALWAYS);
                } else if (sbi->blkszbits != PAGE_SHIFT) {
                        errorfc(fc, "unsupported blocksize for DAX");
                        clear_opt(&sbi->opt, DAX_ALWAYS);
                }
        }

        sb->s_time_gran = 1;
        sb->s_xattr = erofs_xattr_handlers;
        sb->s_export_op = &erofs_export_ops;

        if (test_opt(&sbi->opt, POSIX_ACL))
                sb->s_flags |= SB_POSIXACL;
        else
                sb->s_flags &= ~SB_POSIXACL;

#ifdef CONFIG_EROFS_FS_ZIP
        xa_init(&sbi->managed_pslots);
#endif

        inode = erofs_iget(sb, sbi->root_nid);
        if (IS_ERR(inode))
                return PTR_ERR(inode);

        if (!S_ISDIR(inode->i_mode)) {
                erofs_err(sb, "rootino(nid %llu) is not a directory(i_mode %o)",
                          sbi->root_nid, inode->i_mode);
                iput(inode);
                return -EINVAL;
        }

        sb->s_root = d_make_root(inode);
        if (!sb->s_root)
                return -ENOMEM;

        erofs_shrinker_register(sb);
        if (erofs_sb_has_fragments(sbi) && sbi->packed_nid) {
                sbi->packed_inode = erofs_iget(sb, sbi->packed_nid);
                if (IS_ERR(sbi->packed_inode)) {
                        err = PTR_ERR(sbi->packed_inode);
                        sbi->packed_inode = NULL;
                        return err;
                }
        }
        err = erofs_init_managed_cache(sb);
        if (err)
                return err;

        err = erofs_xattr_prefixes_init(sb);
        if (err)
                return err;

        erofs_set_sysfs_name(sb);
        err = erofs_register_sysfs(sb);
        if (err)
                return err;

        erofs_info(sb, "mounted with root inode @ nid %llu.", sbi->root_nid);
        return 0;
}

static int erofs_fc_get_tree(struct fs_context *fc)
{
        struct erofs_sb_info *sbi = fc->s_fs_info;
        int ret;

        if (IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && sbi->fsid)
                return get_tree_nodev(fc, erofs_fc_fill_super);

        ret = get_tree_bdev_flags(fc, erofs_fc_fill_super,
                IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) ?
                        GET_TREE_BDEV_QUIET_LOOKUP : 0);
#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE
        if (ret == -ENOTBLK) {
                if (!fc->source)
                        return invalf(fc, "No source specified");
                sbi->fdev = filp_open(fc->source, O_RDONLY | O_LARGEFILE, 0);
                if (IS_ERR(sbi->fdev))
                        return PTR_ERR(sbi->fdev);

                if (S_ISREG(file_inode(sbi->fdev)->i_mode) &&
                    sbi->fdev->f_mapping->a_ops->read_folio)
                        return get_tree_nodev(fc, erofs_fc_fill_super);
                fput(sbi->fdev);
        }
#endif
        return ret;
}

static int erofs_fc_reconfigure(struct fs_context *fc)
{
        struct super_block *sb = fc->root->d_sb;
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        struct erofs_sb_info *new_sbi = fc->s_fs_info;

        DBG_BUGON(!sb_rdonly(sb));

        if (new_sbi->fsid || new_sbi->domain_id)
                erofs_info(sb, "ignoring reconfiguration for fsid|domain_id.");

        if (test_opt(&new_sbi->opt, POSIX_ACL))
                fc->sb_flags |= SB_POSIXACL;
        else
                fc->sb_flags &= ~SB_POSIXACL;

        sbi->opt = new_sbi->opt;

        fc->sb_flags |= SB_RDONLY;
        return 0;
}

static int erofs_release_device_info(int id, void *ptr, void *data)
{
        struct erofs_device_info *dif = ptr;

        fs_put_dax(dif->dax_dev, NULL);
        if (dif->file)
                fput(dif->file);
        erofs_fscache_unregister_cookie(dif->fscache);
        dif->fscache = NULL;
        kfree(dif->path);
        kfree(dif);
        return 0;
}

static void erofs_free_dev_context(struct erofs_dev_context *devs)
{
        if (!devs)
                return;
        idr_for_each(&devs->tree, &erofs_release_device_info, NULL);
        idr_destroy(&devs->tree);
        kfree(devs);
}

static void erofs_fc_free(struct fs_context *fc)
{
        struct erofs_sb_info *sbi = fc->s_fs_info;

        if (!sbi)
                return;

        erofs_free_dev_context(sbi->devs);
        kfree(sbi->fsid);
        kfree(sbi->domain_id);
        kfree(sbi);
}

static const struct fs_context_operations erofs_context_ops = {
        .parse_param        = erofs_fc_parse_param,
        .get_tree       = erofs_fc_get_tree,
        .reconfigure    = erofs_fc_reconfigure,
        .free                = erofs_fc_free,
};

static int erofs_init_fs_context(struct fs_context *fc)
{
        struct erofs_sb_info *sbi;

        sbi = kzalloc(sizeof(*sbi), GFP_KERNEL);
        if (!sbi)
                return -ENOMEM;

        sbi->devs = kzalloc(sizeof(struct erofs_dev_context), GFP_KERNEL);
        if (!sbi->devs) {
                kfree(sbi);
                return -ENOMEM;
        }
        fc->s_fs_info = sbi;

        idr_init(&sbi->devs->tree);
        init_rwsem(&sbi->devs->rwsem);
        erofs_default_options(sbi);
        fc->ops = &erofs_context_ops;
        return 0;
}

static void erofs_kill_sb(struct super_block *sb)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);

        if ((IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && sbi->fsid) || sbi->fdev)
                kill_anon_super(sb);
        else
                kill_block_super(sb);

        erofs_free_dev_context(sbi->devs);
        fs_put_dax(sbi->dax_dev, NULL);
        erofs_fscache_unregister_fs(sb);
        kfree(sbi->fsid);
        kfree(sbi->domain_id);
        if (sbi->fdev)
                fput(sbi->fdev);
        kfree(sbi);
        sb->s_fs_info = NULL;
}

static void erofs_put_super(struct super_block *sb)
{
        struct erofs_sb_info *const sbi = EROFS_SB(sb);

        DBG_BUGON(!sbi);

        erofs_unregister_sysfs(sb);
        erofs_shrinker_unregister(sb);
        erofs_xattr_prefixes_cleanup(sb);
#ifdef CONFIG_EROFS_FS_ZIP
        iput(sbi->managed_cache);
        sbi->managed_cache = NULL;
#endif
        iput(sbi->packed_inode);
        sbi->packed_inode = NULL;
        erofs_free_dev_context(sbi->devs);
        sbi->devs = NULL;
        erofs_fscache_unregister_fs(sb);
}

static struct file_system_type erofs_fs_type = {
        .owner          = THIS_MODULE,
        .name           = "erofs",
        .init_fs_context = erofs_init_fs_context,
        .kill_sb        = erofs_kill_sb,
        .fs_flags       = FS_REQUIRES_DEV | FS_ALLOW_IDMAP,
};
MODULE_ALIAS_FS("erofs");

static int __init erofs_module_init(void)
{
        int err;

        erofs_check_ondisk_layout_definitions();

        erofs_inode_cachep = kmem_cache_create("erofs_inode",
                        sizeof(struct erofs_inode), 0,
                        SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,
                        erofs_inode_init_once);
        if (!erofs_inode_cachep)
                return -ENOMEM;

        err = erofs_init_shrinker();
        if (err)
                goto shrinker_err;

        err = z_erofs_init_subsystem();
        if (err)
                goto zip_err;

        err = erofs_init_sysfs();
        if (err)
                goto sysfs_err;

        err = register_filesystem(&erofs_fs_type);
        if (err)
                goto fs_err;

        return 0;

fs_err:
        erofs_exit_sysfs();
sysfs_err:
        z_erofs_exit_subsystem();
zip_err:
        erofs_exit_shrinker();
shrinker_err:
        kmem_cache_destroy(erofs_inode_cachep);
        return err;
}

static void __exit erofs_module_exit(void)
{
        unregister_filesystem(&erofs_fs_type);

        /* Ensure all RCU free inodes / pclusters are safe to be destroyed. */
        rcu_barrier();

        erofs_exit_sysfs();
        z_erofs_exit_subsystem();
        erofs_exit_shrinker();
        kmem_cache_destroy(erofs_inode_cachep);
}

static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf)
{
        struct super_block *sb = dentry->d_sb;
        struct erofs_sb_info *sbi = EROFS_SB(sb);

        buf->f_type = sb->s_magic;
        buf->f_bsize = sb->s_blocksize;
        buf->f_blocks = sbi->total_blocks;
        buf->f_bfree = buf->f_bavail = 0;
        buf->f_files = ULLONG_MAX;
        buf->f_ffree = ULLONG_MAX - sbi->inos;
        buf->f_namelen = EROFS_NAME_LEN;

        if (uuid_is_null(&sb->s_uuid))
                buf->f_fsid = u64_to_fsid(!sb->s_bdev ? 0 :
                                huge_encode_dev(sb->s_bdev->bd_dev));
        else
                buf->f_fsid = uuid_to_fsid(sb->s_uuid.b);
        return 0;
}

static int erofs_show_options(struct seq_file *seq, struct dentry *root)
{
        struct erofs_sb_info *sbi = EROFS_SB(root->d_sb);
        struct erofs_mount_opts *opt = &sbi->opt;

        if (IS_ENABLED(CONFIG_EROFS_FS_XATTR))
                seq_puts(seq, test_opt(opt, XATTR_USER) ?
                                ",user_xattr" : ",nouser_xattr");
        if (IS_ENABLED(CONFIG_EROFS_FS_POSIX_ACL))
                seq_puts(seq, test_opt(opt, POSIX_ACL) ? ",acl" : ",noacl");
        if (IS_ENABLED(CONFIG_EROFS_FS_ZIP))
                seq_printf(seq, ",cache_strategy=%s",
                          erofs_param_cache_strategy[opt->cache_strategy].name);
        if (test_opt(opt, DAX_ALWAYS))
                seq_puts(seq, ",dax=always");
        if (test_opt(opt, DAX_NEVER))
                seq_puts(seq, ",dax=never");
#ifdef CONFIG_EROFS_FS_ONDEMAND
        if (sbi->fsid)
                seq_printf(seq, ",fsid=%s", sbi->fsid);
        if (sbi->domain_id)
                seq_printf(seq, ",domain_id=%s", sbi->domain_id);
#endif
        return 0;
}

const struct super_operations erofs_sops = {
        .put_super = erofs_put_super,
        .alloc_inode = erofs_alloc_inode,
        .free_inode = erofs_free_inode,
        .statfs = erofs_statfs,
        .show_options = erofs_show_options,
};

module_init(erofs_module_init);
module_exit(erofs_module_exit);

MODULE_DESCRIPTION("Enhanced ROM File System");
MODULE_AUTHOR("Gao Xiang, Chao Yu, Miao Xie, CONSUMER BG, HUAWEI Inc.");
MODULE_LICENSE("GPL");








































   11 


   11 

   11 














































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
// SPDX-License-Identifier: GPL-2.0-only
/*
 * linux/net/sunrpc/timer.c
 *
 * Estimate RPC request round trip time.
 *
 * Based on packet round-trip and variance estimator algorithms described
 * in appendix A of "Congestion Avoidance and Control" by Van Jacobson
 * and Michael J. Karels (ACM Computer Communication Review; Proceedings
 * of the Sigcomm '88 Symposium in Stanford, CA, August, 1988).
 *
 * This RTT estimator is used only for RPC over datagram protocols.
 *
 * Copyright (C) 2002 Trond Myklebust <trond.myklebust@fys.uio.no>
 */

#include <asm/param.h>

#include <linux/types.h>
#include <linux/unistd.h>
#include <linux/module.h>

#include <linux/sunrpc/clnt.h>

#define RPC_RTO_MAX (60*HZ)
#define RPC_RTO_INIT (HZ/5)
#define RPC_RTO_MIN (HZ/10)

/**
 * rpc_init_rtt - Initialize an RPC RTT estimator context
 * @rt: context to initialize
 * @timeo: initial timeout value, in jiffies
 *
 */
void rpc_init_rtt(struct rpc_rtt *rt, unsigned long timeo)
{
        unsigned long init = 0;
        unsigned int i;

        rt->timeo = timeo;

        if (timeo > RPC_RTO_INIT)
                init = (timeo - RPC_RTO_INIT) << 3;
        for (i = 0; i < 5; i++) {
                rt->srtt[i] = init;
                rt->sdrtt[i] = RPC_RTO_INIT;
                rt->ntimeouts[i] = 0;
        }
}
EXPORT_SYMBOL_GPL(rpc_init_rtt);

/**
 * rpc_update_rtt - Update an RPC RTT estimator context
 * @rt: context to update
 * @timer: timer array index (request type)
 * @m: recent actual RTT, in jiffies
 *
 * NB: When computing the smoothed RTT and standard deviation,
 *     be careful not to produce negative intermediate results.
 */
void rpc_update_rtt(struct rpc_rtt *rt, unsigned int timer, long m)
{
        long *srtt, *sdrtt;

        if (timer-- == 0)
                return;

        /* jiffies wrapped; ignore this one */
        if (m < 0)
                return;

        if (m == 0)
                m = 1L;

        srtt = (long *)&rt->srtt[timer];
        m -= *srtt >> 3;
        *srtt += m;

        if (m < 0)
                m = -m;

        sdrtt = (long *)&rt->sdrtt[timer];
        m -= *sdrtt >> 2;
        *sdrtt += m;

        /* Set lower bound on the variance */
        if (*sdrtt < RPC_RTO_MIN)
                *sdrtt = RPC_RTO_MIN;
}
EXPORT_SYMBOL_GPL(rpc_update_rtt);

/**
 * rpc_calc_rto - Provide an estimated timeout value
 * @rt: context to use for calculation
 * @timer: timer array index (request type)
 *
 * Estimate RTO for an NFS RPC sent via an unreliable datagram.  Use
 * the mean and mean deviation of RTT for the appropriate type of RPC
 * for frequently issued RPCs, and a fixed default for the others.
 *
 * The justification for doing "other" this way is that these RPCs
 * happen so infrequently that timer estimation would probably be
 * stale.  Also, since many of these RPCs are non-idempotent, a
 * conservative timeout is desired.
 *
 * getattr, lookup,
 * read, write, commit     - A+4D
 * other                   - timeo
 */
unsigned long rpc_calc_rto(struct rpc_rtt *rt, unsigned int timer)
{
        unsigned long res;

        if (timer-- == 0)
                return rt->timeo;

        res = ((rt->srtt[timer] + 7) >> 3) + rt->sdrtt[timer];
        if (res > RPC_RTO_MAX)
                res = RPC_RTO_MAX;

        return res;
}
EXPORT_SYMBOL_GPL(rpc_calc_rto);





























 1844 





 1833 



















   94 

   94 



   80 


   95 




















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Lock-less NULL terminated single linked list
 *
 * The basic atomic operation of this list is cmpxchg on long.  On
 * architectures that don't have NMI-safe cmpxchg implementation, the
 * list can NOT be used in NMI handlers.  So code that uses the list in
 * an NMI handler should depend on CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG.
 *
 * Copyright 2010,2011 Intel Corp.
 *   Author: Huang Ying <ying.huang@intel.com>
 */
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/llist.h>


/**
 * llist_add_batch - add several linked entries in batch
 * @new_first:        first entry in batch to be added
 * @new_last:        last entry in batch to be added
 * @head:        the head for your lock-less list
 *
 * Return whether list is empty before adding.
 */
bool llist_add_batch(struct llist_node *new_first, struct llist_node *new_last,
                     struct llist_head *head)
{
        struct llist_node *first = READ_ONCE(head->first);

        do {
                new_last->next = first;
        } while (!try_cmpxchg(&head->first, &first, new_first));

        return !first;
}
EXPORT_SYMBOL_GPL(llist_add_batch);

/**
 * llist_del_first - delete the first entry of lock-less list
 * @head:        the head for your lock-less list
 *
 * If list is empty, return NULL, otherwise, return the first entry
 * deleted, this is the newest added one.
 *
 * Only one llist_del_first user can be used simultaneously with
 * multiple llist_add users without lock.  Because otherwise
 * llist_del_first, llist_add, llist_add (or llist_del_all, llist_add,
 * llist_add) sequence in another user may change @head->first->next,
 * but keep @head->first.  If multiple consumers are needed, please
 * use llist_del_all or use lock between consumers.
 */
struct llist_node *llist_del_first(struct llist_head *head)
{
        struct llist_node *entry, *next;

        entry = smp_load_acquire(&head->first);
        do {
                if (entry == NULL)
                        return NULL;
                next = READ_ONCE(entry->next);
        } while (!try_cmpxchg(&head->first, &entry, next));

        return entry;
}
EXPORT_SYMBOL_GPL(llist_del_first);

/**
 * llist_del_first_this - delete given entry of lock-less list if it is first
 * @head:        the head for your lock-less list
 * @this:        a list entry.
 *
 * If head of the list is given entry, delete and return %true else
 * return %false.
 *
 * Multiple callers can safely call this concurrently with multiple
 * llist_add() callers, providing all the callers offer a different @this.
 */
bool llist_del_first_this(struct llist_head *head,
                          struct llist_node *this)
{
        struct llist_node *entry, *next;

        /* acquire ensures orderig wrt try_cmpxchg() is llist_del_first() */
        entry = smp_load_acquire(&head->first);
        do {
                if (entry != this)
                        return false;
                next = READ_ONCE(entry->next);
        } while (!try_cmpxchg(&head->first, &entry, next));

        return true;
}
EXPORT_SYMBOL_GPL(llist_del_first_this);

/**
 * llist_reverse_order - reverse order of a llist chain
 * @head:        first item of the list to be reversed
 *
 * Reverse the order of a chain of llist entries and return the
 * new first entry.
 */
struct llist_node *llist_reverse_order(struct llist_node *head)
{
        struct llist_node *new_head = NULL;

        while (head) {
                struct llist_node *tmp = head;
                head = head->next;
                tmp->next = new_head;
                new_head = tmp;
        }

        return new_head;
}
EXPORT_SYMBOL_GPL(llist_reverse_order);



















































































































































































































































































   68 

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
// SPDX-License-Identifier: GPL-2.0-only
/* (C) 1999-2001 Paul `Rusty' Russell
 * (C) 2002-2004 Netfilter Core Team <coreteam@netfilter.org>
 * (C) 2006-2012 Patrick McHardy <kaber@trash.net>
 */

#include <linux/types.h>
#include <linux/timer.h>
#include <linux/module.h>
#include <linux/udp.h>
#include <linux/seq_file.h>
#include <linux/skbuff.h>
#include <linux/ipv6.h>
#include <net/ip6_checksum.h>
#include <net/checksum.h>

#include <linux/netfilter.h>
#include <linux/netfilter_ipv4.h>
#include <linux/netfilter_ipv6.h>
#include <net/netfilter/nf_conntrack_l4proto.h>
#include <net/netfilter/nf_conntrack_ecache.h>
#include <net/netfilter/nf_conntrack_timeout.h>
#include <net/netfilter/nf_log.h>
#include <net/netfilter/ipv4/nf_conntrack_ipv4.h>
#include <net/netfilter/ipv6/nf_conntrack_ipv6.h>

static const unsigned int udp_timeouts[UDP_CT_MAX] = {
        [UDP_CT_UNREPLIED]        = 30*HZ,
        [UDP_CT_REPLIED]        = 120*HZ,
};

static unsigned int *udp_get_timeouts(struct net *net)
{
        return nf_udp_pernet(net)->timeouts;
}

static void udp_error_log(const struct sk_buff *skb,
                          const struct nf_hook_state *state,
                          const char *msg)
{
        nf_l4proto_log_invalid(skb, state, IPPROTO_UDP, "%s", msg);
}

static bool udp_error(struct sk_buff *skb,
                      unsigned int dataoff,
                      const struct nf_hook_state *state)
{
        unsigned int udplen = skb->len - dataoff;
        const struct udphdr *hdr;
        struct udphdr _hdr;

        /* Header is too small? */
        hdr = skb_header_pointer(skb, dataoff, sizeof(_hdr), &_hdr);
        if (!hdr) {
                udp_error_log(skb, state, "short packet");
                return true;
        }

        /* Truncated/malformed packets */
        if (ntohs(hdr->len) > udplen || ntohs(hdr->len) < sizeof(*hdr)) {
                udp_error_log(skb, state, "truncated/malformed packet");
                return true;
        }

        /* Packet with no checksum */
        if (!hdr->check)
                return false;

        /* Checksum invalid? Ignore.
         * We skip checking packets on the outgoing path
         * because the checksum is assumed to be correct.
         * FIXME: Source route IP option packets --RR */
        if (state->hook == NF_INET_PRE_ROUTING &&
            state->net->ct.sysctl_checksum &&
            nf_checksum(skb, state->hook, dataoff, IPPROTO_UDP, state->pf)) {
                udp_error_log(skb, state, "bad checksum");
                return true;
        }

        return false;
}

/* Returns verdict for packet, and may modify conntracktype */
int nf_conntrack_udp_packet(struct nf_conn *ct,
                            struct sk_buff *skb,
                            unsigned int dataoff,
                            enum ip_conntrack_info ctinfo,
                            const struct nf_hook_state *state)
{
        unsigned int *timeouts;
        unsigned long status;

        if (udp_error(skb, dataoff, state))
                return -NF_ACCEPT;

        timeouts = nf_ct_timeout_lookup(ct);
        if (!timeouts)
                timeouts = udp_get_timeouts(nf_ct_net(ct));

        status = READ_ONCE(ct->status);
        if ((status & IPS_CONFIRMED) == 0)
                ct->proto.udp.stream_ts = 2 * HZ + jiffies;

        /* If we've seen traffic both ways, this is some kind of UDP
         * stream. Set Assured.
         */
        if (status & IPS_SEEN_REPLY) {
                unsigned long extra = timeouts[UDP_CT_UNREPLIED];
                bool stream = false;

                /* Still active after two seconds? Extend timeout. */
                if (time_after(jiffies, ct->proto.udp.stream_ts)) {
                        extra = timeouts[UDP_CT_REPLIED];
                        stream = (status & IPS_ASSURED) == 0;
                }

                nf_ct_refresh_acct(ct, ctinfo, skb, extra);

                /* never set ASSURED for IPS_NAT_CLASH, they time out soon */
                if (unlikely((status & IPS_NAT_CLASH)))
                        return NF_ACCEPT;

                /* Also, more likely to be important, and not a probe */
                if (stream && !test_and_set_bit(IPS_ASSURED_BIT, &ct->status))
                        nf_conntrack_event_cache(IPCT_ASSURED, ct);
        } else {
                nf_ct_refresh_acct(ct, ctinfo, skb, timeouts[UDP_CT_UNREPLIED]);
        }
        return NF_ACCEPT;
}

#ifdef CONFIG_NF_CT_PROTO_UDPLITE
static void udplite_error_log(const struct sk_buff *skb,
                              const struct nf_hook_state *state,
                              const char *msg)
{
        nf_l4proto_log_invalid(skb, state, IPPROTO_UDPLITE, "%s", msg);
}

static bool udplite_error(struct sk_buff *skb,
                          unsigned int dataoff,
                          const struct nf_hook_state *state)
{
        unsigned int udplen = skb->len - dataoff;
        const struct udphdr *hdr;
        struct udphdr _hdr;
        unsigned int cscov;

        /* Header is too small? */
        hdr = skb_header_pointer(skb, dataoff, sizeof(_hdr), &_hdr);
        if (!hdr) {
                udplite_error_log(skb, state, "short packet");
                return true;
        }

        cscov = ntohs(hdr->len);
        if (cscov == 0) {
                cscov = udplen;
        } else if (cscov < sizeof(*hdr) || cscov > udplen) {
                udplite_error_log(skb, state, "invalid checksum coverage");
                return true;
        }

        /* UDPLITE mandates checksums */
        if (!hdr->check) {
                udplite_error_log(skb, state, "checksum missing");
                return true;
        }

        /* Checksum invalid? Ignore. */
        if (state->hook == NF_INET_PRE_ROUTING &&
            state->net->ct.sysctl_checksum &&
            nf_checksum_partial(skb, state->hook, dataoff, cscov, IPPROTO_UDP,
                                state->pf)) {
                udplite_error_log(skb, state, "bad checksum");
                return true;
        }

        return false;
}

/* Returns verdict for packet, and may modify conntracktype */
int nf_conntrack_udplite_packet(struct nf_conn *ct,
                                struct sk_buff *skb,
                                unsigned int dataoff,
                                enum ip_conntrack_info ctinfo,
                                const struct nf_hook_state *state)
{
        unsigned int *timeouts;

        if (udplite_error(skb, dataoff, state))
                return -NF_ACCEPT;

        timeouts = nf_ct_timeout_lookup(ct);
        if (!timeouts)
                timeouts = udp_get_timeouts(nf_ct_net(ct));

        /* If we've seen traffic both ways, this is some kind of UDP
           stream.  Extend timeout. */
        if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
                nf_ct_refresh_acct(ct, ctinfo, skb,
                                   timeouts[UDP_CT_REPLIED]);

                if (unlikely((ct->status & IPS_NAT_CLASH)))
                        return NF_ACCEPT;

                /* Also, more likely to be important, and not a probe */
                if (!test_and_set_bit(IPS_ASSURED_BIT, &ct->status))
                        nf_conntrack_event_cache(IPCT_ASSURED, ct);
        } else {
                nf_ct_refresh_acct(ct, ctinfo, skb, timeouts[UDP_CT_UNREPLIED]);
        }
        return NF_ACCEPT;
}
#endif

#ifdef CONFIG_NF_CONNTRACK_TIMEOUT

#include <linux/netfilter/nfnetlink.h>
#include <linux/netfilter/nfnetlink_cttimeout.h>

static int udp_timeout_nlattr_to_obj(struct nlattr *tb[],
                                     struct net *net, void *data)
{
        unsigned int *timeouts = data;
        struct nf_udp_net *un = nf_udp_pernet(net);

        if (!timeouts)
                timeouts = un->timeouts;

        /* set default timeouts for UDP. */
        timeouts[UDP_CT_UNREPLIED] = un->timeouts[UDP_CT_UNREPLIED];
        timeouts[UDP_CT_REPLIED] = un->timeouts[UDP_CT_REPLIED];

        if (tb[CTA_TIMEOUT_UDP_UNREPLIED]) {
                timeouts[UDP_CT_UNREPLIED] =
                        ntohl(nla_get_be32(tb[CTA_TIMEOUT_UDP_UNREPLIED])) * HZ;
        }
        if (tb[CTA_TIMEOUT_UDP_REPLIED]) {
                timeouts[UDP_CT_REPLIED] =
                        ntohl(nla_get_be32(tb[CTA_TIMEOUT_UDP_REPLIED])) * HZ;
        }
        return 0;
}

static int
udp_timeout_obj_to_nlattr(struct sk_buff *skb, const void *data)
{
        const unsigned int *timeouts = data;

        if (nla_put_be32(skb, CTA_TIMEOUT_UDP_UNREPLIED,
                         htonl(timeouts[UDP_CT_UNREPLIED] / HZ)) ||
            nla_put_be32(skb, CTA_TIMEOUT_UDP_REPLIED,
                         htonl(timeouts[UDP_CT_REPLIED] / HZ)))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -ENOSPC;
}

static const struct nla_policy
udp_timeout_nla_policy[CTA_TIMEOUT_UDP_MAX+1] = {
       [CTA_TIMEOUT_UDP_UNREPLIED]        = { .type = NLA_U32 },
       [CTA_TIMEOUT_UDP_REPLIED]        = { .type = NLA_U32 },
};
#endif /* CONFIG_NF_CONNTRACK_TIMEOUT */

void nf_conntrack_udp_init_net(struct net *net)
{
        struct nf_udp_net *un = nf_udp_pernet(net);
        int i;

        for (i = 0; i < UDP_CT_MAX; i++)
                un->timeouts[i] = udp_timeouts[i];

#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
        un->offload_timeout = 30 * HZ;
#endif
}

const struct nf_conntrack_l4proto nf_conntrack_l4proto_udp =
{
        .l4proto                = IPPROTO_UDP,
        .allow_clash                = true,
#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
        .tuple_to_nlattr        = nf_ct_port_tuple_to_nlattr,
        .nlattr_to_tuple        = nf_ct_port_nlattr_to_tuple,
        .nlattr_tuple_size        = nf_ct_port_nlattr_tuple_size,
        .nla_policy                = nf_ct_port_nla_policy,
#endif
#ifdef CONFIG_NF_CONNTRACK_TIMEOUT
        .ctnl_timeout                = {
                .nlattr_to_obj        = udp_timeout_nlattr_to_obj,
                .obj_to_nlattr        = udp_timeout_obj_to_nlattr,
                .nlattr_max        = CTA_TIMEOUT_UDP_MAX,
                .obj_size        = sizeof(unsigned int) * CTA_TIMEOUT_UDP_MAX,
                .nla_policy        = udp_timeout_nla_policy,
        },
#endif /* CONFIG_NF_CONNTRACK_TIMEOUT */
};

#ifdef CONFIG_NF_CT_PROTO_UDPLITE
const struct nf_conntrack_l4proto nf_conntrack_l4proto_udplite =
{
        .l4proto                = IPPROTO_UDPLITE,
        .allow_clash                = true,
#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
        .tuple_to_nlattr        = nf_ct_port_tuple_to_nlattr,
        .nlattr_to_tuple        = nf_ct_port_nlattr_to_tuple,
        .nlattr_tuple_size        = nf_ct_port_nlattr_tuple_size,
        .nla_policy                = nf_ct_port_nla_policy,
#endif
#ifdef CONFIG_NF_CONNTRACK_TIMEOUT
        .ctnl_timeout                = {
                .nlattr_to_obj        = udp_timeout_nlattr_to_obj,
                .obj_to_nlattr        = udp_timeout_obj_to_nlattr,
                .nlattr_max        = CTA_TIMEOUT_UDP_MAX,
                .obj_size        = sizeof(unsigned int) * CTA_TIMEOUT_UDP_MAX,
                .nla_policy        = udp_timeout_nla_policy,
        },
#endif /* CONFIG_NF_CONNTRACK_TIMEOUT */
};
#endif

































































































  580 

  582 
    4 
  581 
  581 






































   82 
   82 

   82 
   81 






















   77 


   77 
   76 

   15 
   77 









   76 


   76 
   76 


   56 
   76 






   76 
   77 
   77 
   77 























  101 
  100 
  101 


  101 



















   44 


































 3694 

 3682 



 3619 
 3696 

 3688 

 3679 



 3694 

   18 






   18 




   18 
   18 








 2100 
 2101 

 2106 
 2101 
































































 3545 


 3544 


 3547 


 3534 
 3545 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
// SPDX-License-Identifier: GPL-2.0
/*
 *  mm/pgtable-generic.c
 *
 *  Generic pgtable methods declared in linux/pgtable.h
 *
 *  Copyright (C) 2010  Linus Torvalds
 */

#include <linux/pagemap.h>
#include <linux/hugetlb.h>
#include <linux/pgtable.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/mm_inline.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>

/*
 * If a p?d_bad entry is found while walking page tables, report
 * the error, before resetting entry to p?d_none.  Usually (but
 * very seldom) called out from the p?d_none_or_clear_bad macros.
 */

void pgd_clear_bad(pgd_t *pgd)
{
        pgd_ERROR(*pgd);
        pgd_clear(pgd);
}

#ifndef __PAGETABLE_P4D_FOLDED
void p4d_clear_bad(p4d_t *p4d)
{
        p4d_ERROR(*p4d);
        p4d_clear(p4d);
}
#endif

#ifndef __PAGETABLE_PUD_FOLDED
void pud_clear_bad(pud_t *pud)
{
        pud_ERROR(*pud);
        pud_clear(pud);
}
#endif

/*
 * Note that the pmd variant below can't be stub'ed out just as for p4d/pud
 * above. pmd folding is special and typically pmd_* macros refer to upper
 * level even when folded
 */
void pmd_clear_bad(pmd_t *pmd)
{
        pmd_ERROR(*pmd);
        pmd_clear(pmd);
}

#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
/*
 * Only sets the access flags (dirty, accessed), as well as write
 * permission. Furthermore, we know it always gets set to a "more
 * permissive" setting, which allows most architectures to optimize
 * this. We return whether the PTE actually changed, which in turn
 * instructs the caller to do things like update__mmu_cache.  This
 * used to be done in the caller, but sparc needs minor faults to
 * force that call on sun4c so we changed this macro slightly
 */
int ptep_set_access_flags(struct vm_area_struct *vma,
                          unsigned long address, pte_t *ptep,
                          pte_t entry, int dirty)
{
        int changed = !pte_same(ptep_get(ptep), entry);
        if (changed) {
                set_pte_at(vma->vm_mm, address, ptep, entry);
                flush_tlb_fix_spurious_fault(vma, address, ptep);
        }
        return changed;
}
#endif

#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
int ptep_clear_flush_young(struct vm_area_struct *vma,
                           unsigned long address, pte_t *ptep)
{
        int young;
        young = ptep_test_and_clear_young(vma, address, ptep);
        if (young)
                flush_tlb_page(vma, address);
        return young;
}
#endif

#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
                       pte_t *ptep)
{
        struct mm_struct *mm = (vma)->vm_mm;
        pte_t pte;
        pte = ptep_get_and_clear(mm, address, ptep);
        if (pte_accessible(mm, pte))
                flush_tlb_page(vma, address);
        return pte;
}
#endif

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
int pmdp_set_access_flags(struct vm_area_struct *vma,
                          unsigned long address, pmd_t *pmdp,
                          pmd_t entry, int dirty)
{
        int changed = !pmd_same(*pmdp, entry);
        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
        if (changed) {
                set_pmd_at(vma->vm_mm, address, pmdp, entry);
                flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        }
        return changed;
}
#endif

#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
int pmdp_clear_flush_young(struct vm_area_struct *vma,
                           unsigned long address, pmd_t *pmdp)
{
        int young;
        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
        young = pmdp_test_and_clear_young(vma, address, pmdp);
        if (young)
                flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        return young;
}
#endif

#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
                            pmd_t *pmdp)
{
        pmd_t pmd;
        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
        VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) &&
                           !pmd_devmap(*pmdp));
        pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
        flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        return pmd;
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
                            pud_t *pudp)
{
        pud_t pud;

        VM_BUG_ON(address & ~HPAGE_PUD_MASK);
        VM_BUG_ON(!pud_trans_huge(*pudp) && !pud_devmap(*pudp));
        pud = pudp_huge_get_and_clear(vma->vm_mm, address, pudp);
        flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE);
        return pud;
}
#endif
#endif

#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
                                pgtable_t pgtable)
{
        assert_spin_locked(pmd_lockptr(mm, pmdp));

        /* FIFO */
        if (!pmd_huge_pte(mm, pmdp))
                INIT_LIST_HEAD(&pgtable->lru);
        else
                list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
        pmd_huge_pte(mm, pmdp) = pgtable;
}
#endif

#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
/* no "address" argument so destroys page coloring of some arch */
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
{
        pgtable_t pgtable;

        assert_spin_locked(pmd_lockptr(mm, pmdp));

        /* FIFO */
        pgtable = pmd_huge_pte(mm, pmdp);
        pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru,
                                                          struct page, lru);
        if (pmd_huge_pte(mm, pmdp))
                list_del(&pgtable->lru);
        return pgtable;
}
#endif

#ifndef __HAVE_ARCH_PMDP_INVALIDATE
pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
                     pmd_t *pmdp)
{
        VM_WARN_ON_ONCE(!pmd_present(*pmdp));
        pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp));
        flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        return old;
}
#endif

#ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address,
                         pmd_t *pmdp)
{
        VM_WARN_ON_ONCE(!pmd_present(*pmdp));
        return pmdp_invalidate(vma, address, pmdp);
}
#endif

#ifndef pmdp_collapse_flush
pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
                          pmd_t *pmdp)
{
        /*
         * pmd and hugepage pte format are same. So we could
         * use the same function.
         */
        pmd_t pmd;

        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
        VM_BUG_ON(pmd_trans_huge(*pmdp));
        pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);

        /* collapse entails shooting down ptes not pmd */
        flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        return pmd;
}
#endif

/* arch define pte_free_defer in asm/pgalloc.h for its own implementation */
#ifndef pte_free_defer
static void pte_free_now(struct rcu_head *head)
{
        struct page *page;

        page = container_of(head, struct page, rcu_head);
        pte_free(NULL /* mm not passed and not used */, (pgtable_t)page);
}

void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
{
        struct page *page;

        page = pgtable;
        call_rcu(&page->rcu_head, pte_free_now);
}
#endif /* pte_free_defer */
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#if defined(CONFIG_GUP_GET_PXX_LOW_HIGH) && \
        (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RCU))
/*
 * See the comment above ptep_get_lockless() in include/linux/pgtable.h:
 * the barriers in pmdp_get_lockless() cannot guarantee that the value in
 * pmd_high actually belongs with the value in pmd_low; but holding interrupts
 * off blocks the TLB flush between present updates, which guarantees that a
 * successful __pte_offset_map() points to a page from matched halves.
 */
static unsigned long pmdp_get_lockless_start(void)
{
        unsigned long irqflags;

        local_irq_save(irqflags);
        return irqflags;
}
static void pmdp_get_lockless_end(unsigned long irqflags)
{
        local_irq_restore(irqflags);
}
#else
static unsigned long pmdp_get_lockless_start(void) { return 0; }
static void pmdp_get_lockless_end(unsigned long irqflags) { }
#endif

pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp)
{
        unsigned long irqflags;
        pmd_t pmdval;

        rcu_read_lock();
        irqflags = pmdp_get_lockless_start();
        pmdval = pmdp_get_lockless(pmd);
        pmdp_get_lockless_end(irqflags);

        if (pmdvalp)
                *pmdvalp = pmdval;
        if (unlikely(pmd_none(pmdval) || is_pmd_migration_entry(pmdval)))
                goto nomap;
        if (unlikely(pmd_trans_huge(pmdval) || pmd_devmap(pmdval)))
                goto nomap;
        if (unlikely(pmd_bad(pmdval))) {
                pmd_clear_bad(pmd);
                goto nomap;
        }
        return __pte_map(&pmdval, addr);
nomap:
        rcu_read_unlock();
        return NULL;
}

pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd,
                                unsigned long addr, spinlock_t **ptlp)
{
        pmd_t pmdval;
        pte_t *pte;

        pte = __pte_offset_map(pmd, addr, &pmdval);
        if (likely(pte))
                *ptlp = pte_lockptr(mm, &pmdval);
        return pte;
}

pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
                                unsigned long addr, pmd_t *pmdvalp,
                                spinlock_t **ptlp)
{
        pte_t *pte;

        VM_WARN_ON_ONCE(!pmdvalp);
        pte = __pte_offset_map(pmd, addr, pmdvalp);
        if (likely(pte))
                *ptlp = pte_lockptr(mm, pmdvalp);
        return pte;
}

/*
 * pte_offset_map_lock(mm, pmd, addr, ptlp), and its internal implementation
 * __pte_offset_map_lock() below, is usually called with the pmd pointer for
 * addr, reached by walking down the mm's pgd, p4d, pud for addr: either while
 * holding mmap_lock or vma lock for read or for write; or in truncate or rmap
 * context, while holding file's i_mmap_lock or anon_vma lock for read (or for
 * write). In a few cases, it may be used with pmd pointing to a pmd_t already
 * copied to or constructed on the stack.
 *
 * When successful, it returns the pte pointer for addr, with its page table
 * kmapped if necessary (when CONFIG_HIGHPTE), and locked against concurrent
 * modification by software, with a pointer to that spinlock in ptlp (in some
 * configs mm->page_table_lock, in SPLIT_PTLOCK configs a spinlock in table's
 * struct page).  pte_unmap_unlock(pte, ptl) to unlock and unmap afterwards.
 *
 * But it is unsuccessful, returning NULL with *ptlp unchanged, if there is no
 * page table at *pmd: if, for example, the page table has just been removed,
 * or replaced by the huge pmd of a THP.  (When successful, *pmd is rechecked
 * after acquiring the ptlock, and retried internally if it changed: so that a
 * page table can be safely removed or replaced by THP while holding its lock.)
 *
 * pte_offset_map(pmd, addr), and its internal helper __pte_offset_map() above,
 * just returns the pte pointer for addr, its page table kmapped if necessary;
 * or NULL if there is no page table at *pmd.  It does not attempt to lock the
 * page table, so cannot normally be used when the page table is to be updated,
 * or when entries read must be stable.  But it does take rcu_read_lock(): so
 * that even when page table is racily removed, it remains a valid though empty
 * and disconnected table.  Until pte_unmap(pte) unmaps and rcu_read_unlock()s
 * afterwards.
 *
 * pte_offset_map_ro_nolock(mm, pmd, addr, ptlp), above, is like pte_offset_map();
 * but when successful, it also outputs a pointer to the spinlock in ptlp - as
 * pte_offset_map_lock() does, but in this case without locking it.  This helps
 * the caller to avoid a later pte_lockptr(mm, *pmd), which might by that time
 * act on a changed *pmd: pte_offset_map_ro_nolock() provides the correct spinlock
 * pointer for the page table that it returns. Even after grabbing the spinlock,
 * we might be looking either at a page table that is still mapped or one that
 * was unmapped and is about to get freed. But for R/O access this is sufficient.
 * So it is only applicable for read-only cases where any modification operations
 * to the page table are not allowed even if the corresponding spinlock is held
 * afterwards.
 *
 * pte_offset_map_rw_nolock(mm, pmd, addr, pmdvalp, ptlp), above, is like
 * pte_offset_map_ro_nolock(); but when successful, it also outputs the pdmval.
 * It is applicable for may-write cases where any modification operations to the
 * page table may happen after the corresponding spinlock is held afterwards.
 * But the users should make sure the page table is stable like checking pte_same()
 * or checking pmd_same() by using the output pmdval before performing the write
 * operations.
 *
 * Note: "RO" / "RW" expresses the intended semantics, not that the *kmap* will
 * be read-only/read-write protected.
 *
 * Note that free_pgtables(), used after unmapping detached vmas, or when
 * exiting the whole mm, does not take page table lock before freeing a page
 * table, and may not use RCU at all: "outsiders" like khugepaged should avoid
 * pte_offset_map() and co once the vma is detached from mm or mm_users is zero.
 */
pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
                             unsigned long addr, spinlock_t **ptlp)
{
        spinlock_t *ptl;
        pmd_t pmdval;
        pte_t *pte;
again:
        pte = __pte_offset_map(pmd, addr, &pmdval);
        if (unlikely(!pte))
                return pte;
        ptl = pte_lockptr(mm, &pmdval);
        spin_lock(ptl);
        if (likely(pmd_same(pmdval, pmdp_get_lockless(pmd)))) {
                *ptlp = ptl;
                return pte;
        }
        pte_unmap_unlock(pte, ptl);
        goto again;
}























































































































































































































































   76 














   35 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  net/dccp/timer.c
 *
 *  An implementation of the DCCP protocol
 *  Arnaldo Carvalho de Melo <acme@conectiva.com.br>
 */

#include <linux/dccp.h>
#include <linux/skbuff.h>
#include <linux/export.h>

#include "dccp.h"

/* sysctl variables governing numbers of retransmission attempts */
int  sysctl_dccp_request_retries        __read_mostly = TCP_SYN_RETRIES;
int  sysctl_dccp_retries1                __read_mostly = TCP_RETR1;
int  sysctl_dccp_retries2                __read_mostly = TCP_RETR2;

static void dccp_write_err(struct sock *sk)
{
        sk->sk_err = READ_ONCE(sk->sk_err_soft) ? : ETIMEDOUT;
        sk_error_report(sk);

        dccp_send_reset(sk, DCCP_RESET_CODE_ABORTED);
        dccp_done(sk);
        __DCCP_INC_STATS(DCCP_MIB_ABORTONTIMEOUT);
}

/* A write timeout has occurred. Process the after effects. */
static int dccp_write_timeout(struct sock *sk)
{
        const struct inet_connection_sock *icsk = inet_csk(sk);
        int retry_until;

        if (sk->sk_state == DCCP_REQUESTING || sk->sk_state == DCCP_PARTOPEN) {
                if (icsk->icsk_retransmits != 0)
                        dst_negative_advice(sk);
                retry_until = icsk->icsk_syn_retries ?
                            : sysctl_dccp_request_retries;
        } else {
                if (icsk->icsk_retransmits >= sysctl_dccp_retries1) {
                        /* NOTE. draft-ietf-tcpimpl-pmtud-01.txt requires pmtu
                           black hole detection. :-(

                           It is place to make it. It is not made. I do not want
                           to make it. It is disguisting. It does not work in any
                           case. Let me to cite the same draft, which requires for
                           us to implement this:

   "The one security concern raised by this memo is that ICMP black holes
   are often caused by over-zealous security administrators who block
   all ICMP messages.  It is vitally important that those who design and
   deploy security systems understand the impact of strict filtering on
   upper-layer protocols.  The safest web site in the world is worthless
   if most TCP implementations cannot transfer data from it.  It would
   be far nicer to have all of the black holes fixed rather than fixing
   all of the TCP implementations."

                           Golden words :-).
                   */

                        dst_negative_advice(sk);
                }

                retry_until = sysctl_dccp_retries2;
                /*
                 * FIXME: see tcp_write_timout and tcp_out_of_resources
                 */
        }

        if (icsk->icsk_retransmits >= retry_until) {
                /* Has it gone just too far? */
                dccp_write_err(sk);
                return 1;
        }
        return 0;
}

/*
 *        The DCCP retransmit timer.
 */
static void dccp_retransmit_timer(struct sock *sk)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        /*
         * More than 4MSL (8 minutes) has passed, a RESET(aborted) was
         * sent, no need to retransmit, this sock is dead.
         */
        if (dccp_write_timeout(sk))
                return;

        /*
         * We want to know the number of packets retransmitted, not the
         * total number of retransmissions of clones of original packets.
         */
        if (icsk->icsk_retransmits == 0)
                __DCCP_INC_STATS(DCCP_MIB_TIMEOUTS);

        if (dccp_retransmit_skb(sk) != 0) {
                /*
                 * Retransmission failed because of local congestion,
                 * do not backoff.
                 */
                if (--icsk->icsk_retransmits == 0)
                        icsk->icsk_retransmits = 1;
                inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
                                          min(icsk->icsk_rto,
                                              TCP_RESOURCE_PROBE_INTERVAL),
                                          DCCP_RTO_MAX);
                return;
        }

        icsk->icsk_backoff++;

        icsk->icsk_rto = min(icsk->icsk_rto << 1, DCCP_RTO_MAX);
        inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, icsk->icsk_rto,
                                  DCCP_RTO_MAX);
        if (icsk->icsk_retransmits > sysctl_dccp_retries1)
                __sk_dst_reset(sk);
}

static void dccp_write_timer(struct timer_list *t)
{
        struct inet_connection_sock *icsk =
                        from_timer(icsk, t, icsk_retransmit_timer);
        struct sock *sk = &icsk->icsk_inet.sk;
        int event = 0;

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk)) {
                /* Try again later */
                sk_reset_timer(sk, &icsk->icsk_retransmit_timer,
                               jiffies + (HZ / 20));
                goto out;
        }

        if (sk->sk_state == DCCP_CLOSED || !icsk->icsk_pending)
                goto out;

        if (time_after(icsk->icsk_timeout, jiffies)) {
                sk_reset_timer(sk, &icsk->icsk_retransmit_timer,
                               icsk->icsk_timeout);
                goto out;
        }

        event = icsk->icsk_pending;
        icsk->icsk_pending = 0;

        switch (event) {
        case ICSK_TIME_RETRANS:
                dccp_retransmit_timer(sk);
                break;
        }
out:
        bh_unlock_sock(sk);
        sock_put(sk);
}

static void dccp_keepalive_timer(struct timer_list *t)
{
        struct sock *sk = from_timer(sk, t, sk_timer);

        pr_err("dccp should not use a keepalive timer !\n");
        sock_put(sk);
}

/* This is the same as tcp_delack_timer, sans prequeue & mem_reclaim stuff */
static void dccp_delack_timer(struct timer_list *t)
{
        struct inet_connection_sock *icsk =
                        from_timer(icsk, t, icsk_delack_timer);
        struct sock *sk = &icsk->icsk_inet.sk;

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk)) {
                /* Try again later. */
                __NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOCKED);
                sk_reset_timer(sk, &icsk->icsk_delack_timer,
                               jiffies + TCP_DELACK_MIN);
                goto out;
        }

        if (sk->sk_state == DCCP_CLOSED ||
            !(icsk->icsk_ack.pending & ICSK_ACK_TIMER))
                goto out;
        if (time_after(icsk->icsk_ack.timeout, jiffies)) {
                sk_reset_timer(sk, &icsk->icsk_delack_timer,
                               icsk->icsk_ack.timeout);
                goto out;
        }

        icsk->icsk_ack.pending &= ~ICSK_ACK_TIMER;

        if (inet_csk_ack_scheduled(sk)) {
                if (!inet_csk_in_pingpong_mode(sk)) {
                        /* Delayed ACK missed: inflate ATO. */
                        icsk->icsk_ack.ato = min_t(u32, icsk->icsk_ack.ato << 1,
                                                   icsk->icsk_rto);
                } else {
                        /* Delayed ACK missed: leave pingpong mode and
                         * deflate ATO.
                         */
                        inet_csk_exit_pingpong_mode(sk);
                        icsk->icsk_ack.ato = TCP_ATO_MIN;
                }
                dccp_send_ack(sk);
                __NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKS);
        }
out:
        bh_unlock_sock(sk);
        sock_put(sk);
}

/**
 * dccp_write_xmitlet  -  Workhorse for CCID packet dequeueing interface
 * @t: pointer to the tasklet associated with this handler
 *
 * See the comments above %ccid_dequeueing_decision for supported modes.
 */
static void dccp_write_xmitlet(struct tasklet_struct *t)
{
        struct dccp_sock *dp = from_tasklet(dp, t, dccps_xmitlet);
        struct sock *sk = &dp->dccps_inet_connection.icsk_inet.sk;

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk))
                sk_reset_timer(sk, &dccp_sk(sk)->dccps_xmit_timer, jiffies + 1);
        else
                dccp_write_xmit(sk);
        bh_unlock_sock(sk);
        sock_put(sk);
}

static void dccp_write_xmit_timer(struct timer_list *t)
{
        struct dccp_sock *dp = from_timer(dp, t, dccps_xmit_timer);

        dccp_write_xmitlet(&dp->dccps_xmitlet);
}

void dccp_init_xmit_timers(struct sock *sk)
{
        struct dccp_sock *dp = dccp_sk(sk);

        tasklet_setup(&dp->dccps_xmitlet, dccp_write_xmitlet);
        timer_setup(&dp->dccps_xmit_timer, dccp_write_xmit_timer, 0);
        inet_csk_init_xmit_timers(sk, &dccp_write_timer, &dccp_delack_timer,
                                  &dccp_keepalive_timer);
}

static ktime_t dccp_timestamp_seed;
/**
 * dccp_timestamp  -  10s of microseconds time source
 * Returns the number of 10s of microseconds since loading DCCP. This is native
 * DCCP time difference format (RFC 4340, sec. 13).
 * Please note: This will wrap around about circa every 11.9 hours.
 */
u32 dccp_timestamp(void)
{
        u64 delta = (u64)ktime_us_delta(ktime_get_real(), dccp_timestamp_seed);

        do_div(delta, 10);
        return delta;
}
EXPORT_SYMBOL_GPL(dccp_timestamp);

void __init dccp_timestamping_init(void)
{
        dccp_timestamp_seed = ktime_get_real();
}









































  165 




















































  165 




























   39 

















  191 













   68 




































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_CPUSET_H
#define _LINUX_CPUSET_H
/*
 *  cpuset interface
 *
 *  Copyright (C) 2003 BULL SA
 *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
 *
 */

#include <linux/sched.h>
#include <linux/sched/topology.h>
#include <linux/sched/task.h>
#include <linux/cpumask.h>
#include <linux/nodemask.h>
#include <linux/mm.h>
#include <linux/mmu_context.h>
#include <linux/jump_label.h>

#ifdef CONFIG_CPUSETS

/*
 * Static branch rewrites can happen in an arbitrary order for a given
 * key. In code paths where we need to loop with read_mems_allowed_begin() and
 * read_mems_allowed_retry() to get a consistent view of mems_allowed, we need
 * to ensure that begin() always gets rewritten before retry() in the
 * disabled -> enabled transition. If not, then if local irqs are disabled
 * around the loop, we can deadlock since retry() would always be
 * comparing the latest value of the mems_allowed seqcount against 0 as
 * begin() still would see cpusets_enabled() as false. The enabled -> disabled
 * transition should happen in reverse order for the same reasons (want to stop
 * looking at real value of mems_allowed.sequence in retry() first).
 */
extern struct static_key_false cpusets_pre_enable_key;
extern struct static_key_false cpusets_enabled_key;
extern struct static_key_false cpusets_insane_config_key;

static inline bool cpusets_enabled(void)
{
        return static_branch_unlikely(&cpusets_enabled_key);
}

static inline void cpuset_inc(void)
{
        static_branch_inc_cpuslocked(&cpusets_pre_enable_key);
        static_branch_inc_cpuslocked(&cpusets_enabled_key);
}

static inline void cpuset_dec(void)
{
        static_branch_dec_cpuslocked(&cpusets_enabled_key);
        static_branch_dec_cpuslocked(&cpusets_pre_enable_key);
}

/*
 * This will get enabled whenever a cpuset configuration is considered
 * unsupportable in general. E.g. movable only node which cannot satisfy
 * any non movable allocations (see update_nodemask). Page allocator
 * needs to make additional checks for those configurations and this
 * check is meant to guard those checks without any overhead for sane
 * configurations.
 */
static inline bool cpusets_insane_config(void)
{
        return static_branch_unlikely(&cpusets_insane_config_key);
}

extern int cpuset_init(void);
extern void cpuset_init_smp(void);
extern void cpuset_force_rebuild(void);
extern void cpuset_update_active_cpus(void);
extern void inc_dl_tasks_cs(struct task_struct *task);
extern void dec_dl_tasks_cs(struct task_struct *task);
extern void cpuset_lock(void);
extern void cpuset_unlock(void);
extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask);
extern bool cpuset_cpus_allowed_fallback(struct task_struct *p);
extern bool cpuset_cpu_is_isolated(int cpu);
extern nodemask_t cpuset_mems_allowed(struct task_struct *p);
#define cpuset_current_mems_allowed (current->mems_allowed)
void cpuset_init_current_mems_allowed(void);
int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask);

extern bool cpuset_node_allowed(int node, gfp_t gfp_mask);

static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask)
{
        return cpuset_node_allowed(zone_to_nid(z), gfp_mask);
}

static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask)
{
        if (cpusets_enabled())
                return __cpuset_zone_allowed(z, gfp_mask);
        return true;
}

extern int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
                                          const struct task_struct *tsk2);

#ifdef CONFIG_CPUSETS_V1
#define cpuset_memory_pressure_bump()                                 \
        do {                                                        \
                if (cpuset_memory_pressure_enabled)                \
                        __cpuset_memory_pressure_bump();        \
        } while (0)
extern int cpuset_memory_pressure_enabled;
extern void __cpuset_memory_pressure_bump(void);
#else
static inline void cpuset_memory_pressure_bump(void) { }
#endif

extern void cpuset_task_status_allowed(struct seq_file *m,
                                        struct task_struct *task);
extern int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns,
                            struct pid *pid, struct task_struct *tsk);

extern int cpuset_mem_spread_node(void);

static inline int cpuset_do_page_mem_spread(void)
{
        return task_spread_page(current);
}

extern bool current_cpuset_is_being_rebound(void);

extern void rebuild_sched_domains(void);

extern void cpuset_print_current_mems_allowed(void);

/*
 * read_mems_allowed_begin is required when making decisions involving
 * mems_allowed such as during page allocation. mems_allowed can be updated in
 * parallel and depending on the new value an operation can fail potentially
 * causing process failure. A retry loop with read_mems_allowed_begin and
 * read_mems_allowed_retry prevents these artificial failures.
 */
static inline unsigned int read_mems_allowed_begin(void)
{
        if (!static_branch_unlikely(&cpusets_pre_enable_key))
                return 0;

        return read_seqcount_begin(&current->mems_allowed_seq);
}

/*
 * If this returns true, the operation that took place after
 * read_mems_allowed_begin may have failed artificially due to a concurrent
 * update of mems_allowed. It is up to the caller to retry the operation if
 * appropriate.
 */
static inline bool read_mems_allowed_retry(unsigned int seq)
{
        if (!static_branch_unlikely(&cpusets_enabled_key))
                return false;

        return read_seqcount_retry(&current->mems_allowed_seq, seq);
}

static inline void set_mems_allowed(nodemask_t nodemask)
{
        unsigned long flags;

        task_lock(current);
        local_irq_save(flags);
        write_seqcount_begin(&current->mems_allowed_seq);
        current->mems_allowed = nodemask;
        write_seqcount_end(&current->mems_allowed_seq);
        local_irq_restore(flags);
        task_unlock(current);
}

#else /* !CONFIG_CPUSETS */

static inline bool cpusets_enabled(void) { return false; }

static inline bool cpusets_insane_config(void) { return false; }

static inline int cpuset_init(void) { return 0; }
static inline void cpuset_init_smp(void) {}

static inline void cpuset_force_rebuild(void) { }

static inline void cpuset_update_active_cpus(void)
{
        partition_sched_domains(1, NULL, NULL);
}

static inline void inc_dl_tasks_cs(struct task_struct *task) { }
static inline void dec_dl_tasks_cs(struct task_struct *task) { }
static inline void cpuset_lock(void) { }
static inline void cpuset_unlock(void) { }

static inline void cpuset_cpus_allowed(struct task_struct *p,
                                       struct cpumask *mask)
{
        cpumask_copy(mask, task_cpu_possible_mask(p));
}

static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p)
{
        return false;
}

static inline bool cpuset_cpu_is_isolated(int cpu)
{
        return false;
}

static inline nodemask_t cpuset_mems_allowed(struct task_struct *p)
{
        return node_possible_map;
}

#define cpuset_current_mems_allowed (node_states[N_MEMORY])
static inline void cpuset_init_current_mems_allowed(void) {}

static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
{
        return 1;
}

static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask)
{
        return true;
}

static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask)
{
        return true;
}

static inline int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
                                                 const struct task_struct *tsk2)
{
        return 1;
}

static inline void cpuset_memory_pressure_bump(void) {}

static inline void cpuset_task_status_allowed(struct seq_file *m,
                                                struct task_struct *task)
{
}

static inline int cpuset_mem_spread_node(void)
{
        return 0;
}

static inline int cpuset_do_page_mem_spread(void)
{
        return 0;
}

static inline bool current_cpuset_is_being_rebound(void)
{
        return false;
}

static inline void rebuild_sched_domains(void)
{
        partition_sched_domains(1, NULL, NULL);
}

static inline void cpuset_print_current_mems_allowed(void)
{
}

static inline void set_mems_allowed(nodemask_t nodemask)
{
}

static inline unsigned int read_mems_allowed_begin(void)
{
        return 0;
}

static inline bool read_mems_allowed_retry(unsigned int seq)
{
        return false;
}

#endif /* !CONFIG_CPUSETS */

#endif /* _LINUX_CPUSET_H */























































    2 




    2 














    5 





    3 





    3 












































    1 

    1 

    1 
















































































































































































































































































































    2 





    1 

    1 
























    1 
    1 








    5 



    2 

    5 








    4 





    2 

    2 




    1 










    2 
    3 









    5 



    4 










    5 














































































































































































































































































































































































































































































































    1 















    1 



    1 











    1 


































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Minimal file system backend for holding eBPF maps and programs,
 * used by bpf(2) object pinning.
 *
 * Authors:
 *
 *        Daniel Borkmann <daniel@iogearbox.net>
 */

#include <linux/init.h>
#include <linux/magic.h>
#include <linux/major.h>
#include <linux/mount.h>
#include <linux/namei.h>
#include <linux/fs.h>
#include <linux/fs_context.h>
#include <linux/fs_parser.h>
#include <linux/kdev_t.h>
#include <linux/filter.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/kstrtox.h>
#include "preload/bpf_preload.h"

enum bpf_type {
        BPF_TYPE_UNSPEC        = 0,
        BPF_TYPE_PROG,
        BPF_TYPE_MAP,
        BPF_TYPE_LINK,
};

static void *bpf_any_get(void *raw, enum bpf_type type)
{
        switch (type) {
        case BPF_TYPE_PROG:
                bpf_prog_inc(raw);
                break;
        case BPF_TYPE_MAP:
                bpf_map_inc_with_uref(raw);
                break;
        case BPF_TYPE_LINK:
                bpf_link_inc(raw);
                break;
        default:
                WARN_ON_ONCE(1);
                break;
        }

        return raw;
}

static void bpf_any_put(void *raw, enum bpf_type type)
{
        switch (type) {
        case BPF_TYPE_PROG:
                bpf_prog_put(raw);
                break;
        case BPF_TYPE_MAP:
                bpf_map_put_with_uref(raw);
                break;
        case BPF_TYPE_LINK:
                bpf_link_put(raw);
                break;
        default:
                WARN_ON_ONCE(1);
                break;
        }
}

static void *bpf_fd_probe_obj(u32 ufd, enum bpf_type *type)
{
        void *raw;

        raw = bpf_map_get_with_uref(ufd);
        if (!IS_ERR(raw)) {
                *type = BPF_TYPE_MAP;
                return raw;
        }

        raw = bpf_prog_get(ufd);
        if (!IS_ERR(raw)) {
                *type = BPF_TYPE_PROG;
                return raw;
        }

        raw = bpf_link_get_from_fd(ufd);
        if (!IS_ERR(raw)) {
                *type = BPF_TYPE_LINK;
                return raw;
        }

        return ERR_PTR(-EINVAL);
}

static const struct inode_operations bpf_dir_iops;

static const struct inode_operations bpf_prog_iops = { };
static const struct inode_operations bpf_map_iops  = { };
static const struct inode_operations bpf_link_iops  = { };

struct inode *bpf_get_inode(struct super_block *sb,
                            const struct inode *dir,
                            umode_t mode)
{
        struct inode *inode;

        switch (mode & S_IFMT) {
        case S_IFDIR:
        case S_IFREG:
        case S_IFLNK:
                break;
        default:
                return ERR_PTR(-EINVAL);
        }

        inode = new_inode(sb);
        if (!inode)
                return ERR_PTR(-ENOSPC);

        inode->i_ino = get_next_ino();
        simple_inode_init_ts(inode);

        inode_init_owner(&nop_mnt_idmap, inode, dir, mode);

        return inode;
}

static int bpf_inode_type(const struct inode *inode, enum bpf_type *type)
{
        *type = BPF_TYPE_UNSPEC;
        if (inode->i_op == &bpf_prog_iops)
                *type = BPF_TYPE_PROG;
        else if (inode->i_op == &bpf_map_iops)
                *type = BPF_TYPE_MAP;
        else if (inode->i_op == &bpf_link_iops)
                *type = BPF_TYPE_LINK;
        else
                return -EACCES;

        return 0;
}

static void bpf_dentry_finalize(struct dentry *dentry, struct inode *inode,
                                struct inode *dir)
{
        d_instantiate(dentry, inode);
        dget(dentry);

        inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
}

static int bpf_mkdir(struct mnt_idmap *idmap, struct inode *dir,
                     struct dentry *dentry, umode_t mode)
{
        struct inode *inode;

        inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFDIR);
        if (IS_ERR(inode))
                return PTR_ERR(inode);

        inode->i_op = &bpf_dir_iops;
        inode->i_fop = &simple_dir_operations;

        inc_nlink(inode);
        inc_nlink(dir);

        bpf_dentry_finalize(dentry, inode, dir);
        return 0;
}

struct map_iter {
        void *key;
        bool done;
};

static struct map_iter *map_iter(struct seq_file *m)
{
        return m->private;
}

static struct bpf_map *seq_file_to_map(struct seq_file *m)
{
        return file_inode(m->file)->i_private;
}

static void map_iter_free(struct map_iter *iter)
{
        if (iter) {
                kfree(iter->key);
                kfree(iter);
        }
}

static struct map_iter *map_iter_alloc(struct bpf_map *map)
{
        struct map_iter *iter;

        iter = kzalloc(sizeof(*iter), GFP_KERNEL | __GFP_NOWARN);
        if (!iter)
                goto error;

        iter->key = kzalloc(map->key_size, GFP_KERNEL | __GFP_NOWARN);
        if (!iter->key)
                goto error;

        return iter;

error:
        map_iter_free(iter);
        return NULL;
}

static void *map_seq_next(struct seq_file *m, void *v, loff_t *pos)
{
        struct bpf_map *map = seq_file_to_map(m);
        void *key = map_iter(m)->key;
        void *prev_key;

        (*pos)++;
        if (map_iter(m)->done)
                return NULL;

        if (unlikely(v == SEQ_START_TOKEN))
                prev_key = NULL;
        else
                prev_key = key;

        rcu_read_lock();
        if (map->ops->map_get_next_key(map, prev_key, key)) {
                map_iter(m)->done = true;
                key = NULL;
        }
        rcu_read_unlock();
        return key;
}

static void *map_seq_start(struct seq_file *m, loff_t *pos)
{
        if (map_iter(m)->done)
                return NULL;

        return *pos ? map_iter(m)->key : SEQ_START_TOKEN;
}

static void map_seq_stop(struct seq_file *m, void *v)
{
}

static int map_seq_show(struct seq_file *m, void *v)
{
        struct bpf_map *map = seq_file_to_map(m);
        void *key = map_iter(m)->key;

        if (unlikely(v == SEQ_START_TOKEN)) {
                seq_puts(m, "# WARNING!! The output is for debug purpose only\n");
                seq_puts(m, "# WARNING!! The output format will change\n");
        } else {
                map->ops->map_seq_show_elem(map, key, m);
        }

        return 0;
}

static const struct seq_operations bpffs_map_seq_ops = {
        .start        = map_seq_start,
        .next        = map_seq_next,
        .show        = map_seq_show,
        .stop        = map_seq_stop,
};

static int bpffs_map_open(struct inode *inode, struct file *file)
{
        struct bpf_map *map = inode->i_private;
        struct map_iter *iter;
        struct seq_file *m;
        int err;

        iter = map_iter_alloc(map);
        if (!iter)
                return -ENOMEM;

        err = seq_open(file, &bpffs_map_seq_ops);
        if (err) {
                map_iter_free(iter);
                return err;
        }

        m = file->private_data;
        m->private = iter;

        return 0;
}

static int bpffs_map_release(struct inode *inode, struct file *file)
{
        struct seq_file *m = file->private_data;

        map_iter_free(map_iter(m));

        return seq_release(inode, file);
}

/* bpffs_map_fops should only implement the basic
 * read operation for a BPF map.  The purpose is to
 * provide a simple user intuitive way to do
 * "cat bpffs/pathto/a-pinned-map".
 *
 * Other operations (e.g. write, lookup...) should be realized by
 * the userspace tools (e.g. bpftool) through the
 * BPF_OBJ_GET_INFO_BY_FD and the map's lookup/update
 * interface.
 */
static const struct file_operations bpffs_map_fops = {
        .open                = bpffs_map_open,
        .read                = seq_read,
        .release        = bpffs_map_release,
};

static int bpffs_obj_open(struct inode *inode, struct file *file)
{
        return -EIO;
}

static const struct file_operations bpffs_obj_fops = {
        .open                = bpffs_obj_open,
};

static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw,
                         const struct inode_operations *iops,
                         const struct file_operations *fops)
{
        struct inode *dir = dentry->d_parent->d_inode;
        struct inode *inode = bpf_get_inode(dir->i_sb, dir, mode);
        if (IS_ERR(inode))
                return PTR_ERR(inode);

        inode->i_op = iops;
        inode->i_fop = fops;
        inode->i_private = raw;

        bpf_dentry_finalize(dentry, inode, dir);
        return 0;
}

static int bpf_mkprog(struct dentry *dentry, umode_t mode, void *arg)
{
        return bpf_mkobj_ops(dentry, mode, arg, &bpf_prog_iops,
                             &bpffs_obj_fops);
}

static int bpf_mkmap(struct dentry *dentry, umode_t mode, void *arg)
{
        struct bpf_map *map = arg;

        return bpf_mkobj_ops(dentry, mode, arg, &bpf_map_iops,
                             bpf_map_support_seq_show(map) ?
                             &bpffs_map_fops : &bpffs_obj_fops);
}

static int bpf_mklink(struct dentry *dentry, umode_t mode, void *arg)
{
        struct bpf_link *link = arg;

        return bpf_mkobj_ops(dentry, mode, arg, &bpf_link_iops,
                             bpf_link_is_iter(link) ?
                             &bpf_iter_fops : &bpffs_obj_fops);
}

static struct dentry *
bpf_lookup(struct inode *dir, struct dentry *dentry, unsigned flags)
{
        /* Dots in names (e.g. "/sys/fs/bpf/foo.bar") are reserved for future
         * extensions. That allows popoulate_bpffs() create special files.
         */
        if ((dir->i_mode & S_IALLUGO) &&
            strchr(dentry->d_name.name, '.'))
                return ERR_PTR(-EPERM);

        return simple_lookup(dir, dentry, flags);
}

static int bpf_symlink(struct mnt_idmap *idmap, struct inode *dir,
                       struct dentry *dentry, const char *target)
{
        char *link = kstrdup(target, GFP_USER | __GFP_NOWARN);
        struct inode *inode;

        if (!link)
                return -ENOMEM;

        inode = bpf_get_inode(dir->i_sb, dir, S_IRWXUGO | S_IFLNK);
        if (IS_ERR(inode)) {
                kfree(link);
                return PTR_ERR(inode);
        }

        inode->i_op = &simple_symlink_inode_operations;
        inode->i_link = link;

        bpf_dentry_finalize(dentry, inode, dir);
        return 0;
}

static const struct inode_operations bpf_dir_iops = {
        .lookup                = bpf_lookup,
        .mkdir                = bpf_mkdir,
        .symlink        = bpf_symlink,
        .rmdir                = simple_rmdir,
        .rename                = simple_rename,
        .link                = simple_link,
        .unlink                = simple_unlink,
};

/* pin iterator link into bpffs */
static int bpf_iter_link_pin_kernel(struct dentry *parent,
                                    const char *name, struct bpf_link *link)
{
        umode_t mode = S_IFREG | S_IRUSR;
        struct dentry *dentry;
        int ret;

        inode_lock(parent->d_inode);
        dentry = lookup_one_len(name, parent, strlen(name));
        if (IS_ERR(dentry)) {
                inode_unlock(parent->d_inode);
                return PTR_ERR(dentry);
        }
        ret = bpf_mkobj_ops(dentry, mode, link, &bpf_link_iops,
                            &bpf_iter_fops);
        dput(dentry);
        inode_unlock(parent->d_inode);
        return ret;
}

static int bpf_obj_do_pin(int path_fd, const char __user *pathname, void *raw,
                          enum bpf_type type)
{
        struct dentry *dentry;
        struct inode *dir;
        struct path path;
        umode_t mode;
        int ret;

        dentry = user_path_create(path_fd, pathname, &path, 0);
        if (IS_ERR(dentry))
                return PTR_ERR(dentry);

        dir = d_inode(path.dentry);
        if (dir->i_op != &bpf_dir_iops) {
                ret = -EPERM;
                goto out;
        }

        mode = S_IFREG | ((S_IRUSR | S_IWUSR) & ~current_umask());
        ret = security_path_mknod(&path, dentry, mode, 0);
        if (ret)
                goto out;

        switch (type) {
        case BPF_TYPE_PROG:
                ret = vfs_mkobj(dentry, mode, bpf_mkprog, raw);
                break;
        case BPF_TYPE_MAP:
                ret = vfs_mkobj(dentry, mode, bpf_mkmap, raw);
                break;
        case BPF_TYPE_LINK:
                ret = vfs_mkobj(dentry, mode, bpf_mklink, raw);
                break;
        default:
                ret = -EPERM;
        }
out:
        done_path_create(&path, dentry);
        return ret;
}

int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname)
{
        enum bpf_type type;
        void *raw;
        int ret;

        raw = bpf_fd_probe_obj(ufd, &type);
        if (IS_ERR(raw))
                return PTR_ERR(raw);

        ret = bpf_obj_do_pin(path_fd, pathname, raw, type);
        if (ret != 0)
                bpf_any_put(raw, type);

        return ret;
}

static void *bpf_obj_do_get(int path_fd, const char __user *pathname,
                            enum bpf_type *type, int flags)
{
        struct inode *inode;
        struct path path;
        void *raw;
        int ret;

        ret = user_path_at(path_fd, pathname, LOOKUP_FOLLOW, &path);
        if (ret)
                return ERR_PTR(ret);

        inode = d_backing_inode(path.dentry);
        ret = path_permission(&path, ACC_MODE(flags));
        if (ret)
                goto out;

        ret = bpf_inode_type(inode, type);
        if (ret)
                goto out;

        raw = bpf_any_get(inode->i_private, *type);
        if (!IS_ERR(raw))
                touch_atime(&path);

        path_put(&path);
        return raw;
out:
        path_put(&path);
        return ERR_PTR(ret);
}

int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags)
{
        enum bpf_type type = BPF_TYPE_UNSPEC;
        int f_flags;
        void *raw;
        int ret;

        f_flags = bpf_get_file_flag(flags);
        if (f_flags < 0)
                return f_flags;

        raw = bpf_obj_do_get(path_fd, pathname, &type, f_flags);
        if (IS_ERR(raw))
                return PTR_ERR(raw);

        if (type == BPF_TYPE_PROG)
                ret = bpf_prog_new_fd(raw);
        else if (type == BPF_TYPE_MAP)
                ret = bpf_map_new_fd(raw, f_flags);
        else if (type == BPF_TYPE_LINK)
                ret = (f_flags != O_RDWR) ? -EINVAL : bpf_link_new_fd(raw);
        else
                return -ENOENT;

        if (ret < 0)
                bpf_any_put(raw, type);
        return ret;
}

static struct bpf_prog *__get_prog_inode(struct inode *inode, enum bpf_prog_type type)
{
        struct bpf_prog *prog;
        int ret = inode_permission(&nop_mnt_idmap, inode, MAY_READ);
        if (ret)
                return ERR_PTR(ret);

        if (inode->i_op == &bpf_map_iops)
                return ERR_PTR(-EINVAL);
        if (inode->i_op == &bpf_link_iops)
                return ERR_PTR(-EINVAL);
        if (inode->i_op != &bpf_prog_iops)
                return ERR_PTR(-EACCES);

        prog = inode->i_private;

        ret = security_bpf_prog(prog);
        if (ret < 0)
                return ERR_PTR(ret);

        if (!bpf_prog_get_ok(prog, &type, false))
                return ERR_PTR(-EINVAL);

        bpf_prog_inc(prog);
        return prog;
}

struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type)
{
        struct bpf_prog *prog;
        struct path path;
        int ret = kern_path(name, LOOKUP_FOLLOW, &path);
        if (ret)
                return ERR_PTR(ret);
        prog = __get_prog_inode(d_backing_inode(path.dentry), type);
        if (!IS_ERR(prog))
                touch_atime(&path);
        path_put(&path);
        return prog;
}
EXPORT_SYMBOL(bpf_prog_get_type_path);

struct bpffs_btf_enums {
        const struct btf *btf;
        const struct btf_type *cmd_t;
        const struct btf_type *map_t;
        const struct btf_type *prog_t;
        const struct btf_type *attach_t;
};

static int find_bpffs_btf_enums(struct bpffs_btf_enums *info)
{
        const struct btf *btf;
        const struct btf_type *t;
        const char *name;
        int i, n;

        memset(info, 0, sizeof(*info));

        btf = bpf_get_btf_vmlinux();
        if (IS_ERR(btf))
                return PTR_ERR(btf);
        if (!btf)
                return -ENOENT;

        info->btf = btf;

        for (i = 1, n = btf_nr_types(btf); i < n; i++) {
                t = btf_type_by_id(btf, i);
                if (!btf_type_is_enum(t))
                        continue;

                name = btf_name_by_offset(btf, t->name_off);
                if (!name)
                        continue;

                if (strcmp(name, "bpf_cmd") == 0)
                        info->cmd_t = t;
                else if (strcmp(name, "bpf_map_type") == 0)
                        info->map_t = t;
                else if (strcmp(name, "bpf_prog_type") == 0)
                        info->prog_t = t;
                else if (strcmp(name, "bpf_attach_type") == 0)
                        info->attach_t = t;
                else
                        continue;

                if (info->cmd_t && info->map_t && info->prog_t && info->attach_t)
                        return 0;
        }

        return -ESRCH;
}

static bool find_btf_enum_const(const struct btf *btf, const struct btf_type *enum_t,
                                const char *prefix, const char *str, int *value)
{
        const struct btf_enum *e;
        const char *name;
        int i, n, pfx_len = strlen(prefix);

        *value = 0;

        if (!btf || !enum_t)
                return false;

        for (i = 0, n = btf_vlen(enum_t); i < n; i++) {
                e = &btf_enum(enum_t)[i];

                name = btf_name_by_offset(btf, e->name_off);
                if (!name || strncasecmp(name, prefix, pfx_len) != 0)
                        continue;

                /* match symbolic name case insensitive and ignoring prefix */
                if (strcasecmp(name + pfx_len, str) == 0) {
                        *value = e->val;
                        return true;
                }
        }

        return false;
}

static void seq_print_delegate_opts(struct seq_file *m,
                                    const char *opt_name,
                                    const struct btf *btf,
                                    const struct btf_type *enum_t,
                                    const char *prefix,
                                    u64 delegate_msk, u64 any_msk)
{
        const struct btf_enum *e;
        bool first = true;
        const char *name;
        u64 msk;
        int i, n, pfx_len = strlen(prefix);

        delegate_msk &= any_msk; /* clear unknown bits */

        if (delegate_msk == 0)
                return;

        seq_printf(m, ",%s", opt_name);
        if (delegate_msk == any_msk) {
                seq_printf(m, "=any");
                return;
        }

        if (btf && enum_t) {
                for (i = 0, n = btf_vlen(enum_t); i < n; i++) {
                        e = &btf_enum(enum_t)[i];
                        name = btf_name_by_offset(btf, e->name_off);
                        if (!name || strncasecmp(name, prefix, pfx_len) != 0)
                                continue;
                        msk = 1ULL << e->val;
                        if (delegate_msk & msk) {
                                /* emit lower-case name without prefix */
                                seq_putc(m, first ? '=' : ':');
                                name += pfx_len;
                                while (*name) {
                                        seq_putc(m, tolower(*name));
                                        name++;
                                }

                                delegate_msk &= ~msk;
                                first = false;
                        }
                }
        }
        if (delegate_msk)
                seq_printf(m, "%c0x%llx", first ? '=' : ':', delegate_msk);
}

/*
 * Display the mount options in /proc/mounts.
 */
static int bpf_show_options(struct seq_file *m, struct dentry *root)
{
        struct inode *inode = d_inode(root);
        umode_t mode = inode->i_mode & S_IALLUGO & ~S_ISVTX;
        struct bpf_mount_opts *opts = root->d_sb->s_fs_info;
        u64 mask;

        if (!uid_eq(inode->i_uid, GLOBAL_ROOT_UID))
                seq_printf(m, ",uid=%u",
                           from_kuid_munged(&init_user_ns, inode->i_uid));
        if (!gid_eq(inode->i_gid, GLOBAL_ROOT_GID))
                seq_printf(m, ",gid=%u",
                           from_kgid_munged(&init_user_ns, inode->i_gid));
        if (mode != S_IRWXUGO)
                seq_printf(m, ",mode=%o", mode);

        if (opts->delegate_cmds || opts->delegate_maps ||
            opts->delegate_progs || opts->delegate_attachs) {
                struct bpffs_btf_enums info;

                /* ignore errors, fallback to hex */
                (void)find_bpffs_btf_enums(&info);

                mask = (1ULL << __MAX_BPF_CMD) - 1;
                seq_print_delegate_opts(m, "delegate_cmds",
                                        info.btf, info.cmd_t, "BPF_",
                                        opts->delegate_cmds, mask);

                mask = (1ULL << __MAX_BPF_MAP_TYPE) - 1;
                seq_print_delegate_opts(m, "delegate_maps",
                                        info.btf, info.map_t, "BPF_MAP_TYPE_",
                                        opts->delegate_maps, mask);

                mask = (1ULL << __MAX_BPF_PROG_TYPE) - 1;
                seq_print_delegate_opts(m, "delegate_progs",
                                        info.btf, info.prog_t, "BPF_PROG_TYPE_",
                                        opts->delegate_progs, mask);

                mask = (1ULL << __MAX_BPF_ATTACH_TYPE) - 1;
                seq_print_delegate_opts(m, "delegate_attachs",
                                        info.btf, info.attach_t, "BPF_",
                                        opts->delegate_attachs, mask);
        }

        return 0;
}

static void bpf_free_inode(struct inode *inode)
{
        enum bpf_type type;

        if (S_ISLNK(inode->i_mode))
                kfree(inode->i_link);
        if (!bpf_inode_type(inode, &type))
                bpf_any_put(inode->i_private, type);
        free_inode_nonrcu(inode);
}

const struct super_operations bpf_super_ops = {
        .statfs                = simple_statfs,
        .drop_inode        = generic_delete_inode,
        .show_options        = bpf_show_options,
        .free_inode        = bpf_free_inode,
};

enum {
        OPT_UID,
        OPT_GID,
        OPT_MODE,
        OPT_DELEGATE_CMDS,
        OPT_DELEGATE_MAPS,
        OPT_DELEGATE_PROGS,
        OPT_DELEGATE_ATTACHS,
};

static const struct fs_parameter_spec bpf_fs_parameters[] = {
        fsparam_u32        ("uid",                                OPT_UID),
        fsparam_u32        ("gid",                                OPT_GID),
        fsparam_u32oct        ("mode",                        OPT_MODE),
        fsparam_string        ("delegate_cmds",                OPT_DELEGATE_CMDS),
        fsparam_string        ("delegate_maps",                OPT_DELEGATE_MAPS),
        fsparam_string        ("delegate_progs",                OPT_DELEGATE_PROGS),
        fsparam_string        ("delegate_attachs",                OPT_DELEGATE_ATTACHS),
        {}
};

static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param)
{
        struct bpf_mount_opts *opts = fc->s_fs_info;
        struct fs_parse_result result;
        kuid_t uid;
        kgid_t gid;
        int opt, err;

        opt = fs_parse(fc, bpf_fs_parameters, param, &result);
        if (opt < 0) {
                /* We might like to report bad mount options here, but
                 * traditionally we've ignored all mount options, so we'd
                 * better continue to ignore non-existing options for bpf.
                 */
                if (opt == -ENOPARAM) {
                        opt = vfs_parse_fs_param_source(fc, param);
                        if (opt != -ENOPARAM)
                                return opt;

                        return 0;
                }

                if (opt < 0)
                        return opt;
        }

        switch (opt) {
        case OPT_UID:
                uid = make_kuid(current_user_ns(), result.uint_32);
                if (!uid_valid(uid))
                        goto bad_value;

                /*
                 * The requested uid must be representable in the
                 * filesystem's idmapping.
                 */
                if (!kuid_has_mapping(fc->user_ns, uid))
                        goto bad_value;

                opts->uid = uid;
                break;
        case OPT_GID:
                gid = make_kgid(current_user_ns(), result.uint_32);
                if (!gid_valid(gid))
                        goto bad_value;

                /*
                 * The requested gid must be representable in the
                 * filesystem's idmapping.
                 */
                if (!kgid_has_mapping(fc->user_ns, gid))
                        goto bad_value;

                opts->gid = gid;
                break;
        case OPT_MODE:
                opts->mode = result.uint_32 & S_IALLUGO;
                break;
        case OPT_DELEGATE_CMDS:
        case OPT_DELEGATE_MAPS:
        case OPT_DELEGATE_PROGS:
        case OPT_DELEGATE_ATTACHS: {
                struct bpffs_btf_enums info;
                const struct btf_type *enum_t;
                const char *enum_pfx;
                u64 *delegate_msk, msk = 0;
                char *p, *str;
                int val;

                /* ignore errors, fallback to hex */
                (void)find_bpffs_btf_enums(&info);

                switch (opt) {
                case OPT_DELEGATE_CMDS:
                        delegate_msk = &opts->delegate_cmds;
                        enum_t = info.cmd_t;
                        enum_pfx = "BPF_";
                        break;
                case OPT_DELEGATE_MAPS:
                        delegate_msk = &opts->delegate_maps;
                        enum_t = info.map_t;
                        enum_pfx = "BPF_MAP_TYPE_";
                        break;
                case OPT_DELEGATE_PROGS:
                        delegate_msk = &opts->delegate_progs;
                        enum_t = info.prog_t;
                        enum_pfx = "BPF_PROG_TYPE_";
                        break;
                case OPT_DELEGATE_ATTACHS:
                        delegate_msk = &opts->delegate_attachs;
                        enum_t = info.attach_t;
                        enum_pfx = "BPF_";
                        break;
                default:
                        return -EINVAL;
                }

                str = param->string;
                while ((p = strsep(&str, ":"))) {
                        if (strcmp(p, "any") == 0) {
                                msk |= ~0ULL;
                        } else if (find_btf_enum_const(info.btf, enum_t, enum_pfx, p, &val)) {
                                msk |= 1ULL << val;
                        } else {
                                err = kstrtou64(p, 0, &msk);
                                if (err)
                                        return err;
                        }
                }

                /* Setting delegation mount options requires privileges */
                if (msk && !capable(CAP_SYS_ADMIN))
                        return -EPERM;

                *delegate_msk |= msk;
                break;
        }
        default:
                /* ignore unknown mount options */
                break;
        }

        return 0;
bad_value:
        return invalfc(fc, "Bad value for '%s'", param->key);
}

struct bpf_preload_ops *bpf_preload_ops;
EXPORT_SYMBOL_GPL(bpf_preload_ops);

static bool bpf_preload_mod_get(void)
{
        /* If bpf_preload.ko wasn't loaded earlier then load it now.
         * When bpf_preload is built into vmlinux the module's __init
         * function will populate it.
         */
        if (!bpf_preload_ops) {
                request_module("bpf_preload");
                if (!bpf_preload_ops)
                        return false;
        }
        /* And grab the reference, so the module doesn't disappear while the
         * kernel is interacting with the kernel module and its UMD.
         */
        if (!try_module_get(bpf_preload_ops->owner)) {
                pr_err("bpf_preload module get failed.\n");
                return false;
        }
        return true;
}

static void bpf_preload_mod_put(void)
{
        if (bpf_preload_ops)
                /* now user can "rmmod bpf_preload" if necessary */
                module_put(bpf_preload_ops->owner);
}

static DEFINE_MUTEX(bpf_preload_lock);

static int populate_bpffs(struct dentry *parent)
{
        struct bpf_preload_info objs[BPF_PRELOAD_LINKS] = {};
        int err = 0, i;

        /* grab the mutex to make sure the kernel interactions with bpf_preload
         * are serialized
         */
        mutex_lock(&bpf_preload_lock);

        /* if bpf_preload.ko wasn't built into vmlinux then load it */
        if (!bpf_preload_mod_get())
                goto out;

        err = bpf_preload_ops->preload(objs);
        if (err)
                goto out_put;
        for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
                bpf_link_inc(objs[i].link);
                err = bpf_iter_link_pin_kernel(parent,
                                               objs[i].link_name, objs[i].link);
                if (err) {
                        bpf_link_put(objs[i].link);
                        goto out_put;
                }
        }
out_put:
        bpf_preload_mod_put();
out:
        mutex_unlock(&bpf_preload_lock);
        return err;
}

static int bpf_fill_super(struct super_block *sb, struct fs_context *fc)
{
        static const struct tree_descr bpf_rfiles[] = { { "" } };
        struct bpf_mount_opts *opts = sb->s_fs_info;
        struct inode *inode;
        int ret;

        /* Mounting an instance of BPF FS requires privileges */
        if (fc->user_ns != &init_user_ns && !capable(CAP_SYS_ADMIN))
                return -EPERM;

        ret = simple_fill_super(sb, BPF_FS_MAGIC, bpf_rfiles);
        if (ret)
                return ret;

        sb->s_op = &bpf_super_ops;

        inode = sb->s_root->d_inode;
        inode->i_uid = opts->uid;
        inode->i_gid = opts->gid;
        inode->i_op = &bpf_dir_iops;
        inode->i_mode &= ~S_IALLUGO;
        populate_bpffs(sb->s_root);
        inode->i_mode |= S_ISVTX | opts->mode;
        return 0;
}

static int bpf_get_tree(struct fs_context *fc)
{
        return get_tree_nodev(fc, bpf_fill_super);
}

static void bpf_free_fc(struct fs_context *fc)
{
        kfree(fc->s_fs_info);
}

static const struct fs_context_operations bpf_context_ops = {
        .free                = bpf_free_fc,
        .parse_param        = bpf_parse_param,
        .get_tree        = bpf_get_tree,
};

/*
 * Set up the filesystem mount context.
 */
static int bpf_init_fs_context(struct fs_context *fc)
{
        struct bpf_mount_opts *opts;

        opts = kzalloc(sizeof(struct bpf_mount_opts), GFP_KERNEL);
        if (!opts)
                return -ENOMEM;

        opts->mode = S_IRWXUGO;
        opts->uid = current_fsuid();
        opts->gid = current_fsgid();

        /* start out with no BPF token delegation enabled */
        opts->delegate_cmds = 0;
        opts->delegate_maps = 0;
        opts->delegate_progs = 0;
        opts->delegate_attachs = 0;

        fc->s_fs_info = opts;
        fc->ops = &bpf_context_ops;
        return 0;
}

static void bpf_kill_super(struct super_block *sb)
{
        struct bpf_mount_opts *opts = sb->s_fs_info;

        kill_litter_super(sb);
        kfree(opts);
}

static struct file_system_type bpf_fs_type = {
        .owner                = THIS_MODULE,
        .name                = "bpf",
        .init_fs_context = bpf_init_fs_context,
        .parameters        = bpf_fs_parameters,
        .kill_sb        = bpf_kill_super,
        .fs_flags        = FS_USERNS_MOUNT,
};

static int __init bpf_init(void)
{
        int ret;

        ret = sysfs_create_mount_point(fs_kobj, "bpf");
        if (ret)
                return ret;

        ret = register_filesystem(&bpf_fs_type);
        if (ret)
                sysfs_remove_mount_point(fs_kobj, "bpf");

        return ret;
}
fs_initcall(bpf_init);

































































































































































































































   16 




   13 










































    1 













    1 




    1 




    1 

































    3 













    1 

































    3 





    7 







































    3 






















































    6 









































































    1 




















    1 



































































































































































    1 

















































































    4 




    1 




    1 
















































    3 





















































































































    4 













    3 






    3 






    3 













    1 


































































































































    1 





















    4 




























































































































































































    8 






























































































































































































































































    3 



































































































































































    3 
































































































































    3 



















































































































































































































































































































































































    8 





















































































































































































































































































































































































































































































































































































































































































































































































    4 























    4 





































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Portions of this file
 * Copyright(c) 2016-2017 Intel Deutschland GmbH
 * Copyright (C) 2018 - 2024 Intel Corporation
 */

#if !defined(__MAC80211_DRIVER_TRACE) || defined(TRACE_HEADER_MULTI_READ)
#define __MAC80211_DRIVER_TRACE

#include <linux/tracepoint.h>
#include <net/mac80211.h>
#include "ieee80211_i.h"

#undef TRACE_SYSTEM
#define TRACE_SYSTEM mac80211

#define MAXNAME                32
#define LOCAL_ENTRY        __array(char, wiphy_name, 32)
#define LOCAL_ASSIGN        strscpy(__entry->wiphy_name, wiphy_name(local->hw.wiphy), MAXNAME)
#define LOCAL_PR_FMT        "%s"
#define LOCAL_PR_ARG        __entry->wiphy_name

#define STA_ENTRY        __array(char, sta_addr, ETH_ALEN)
#define STA_ASSIGN        (sta ? memcpy(__entry->sta_addr, sta->addr, ETH_ALEN) : \
                                eth_zero_addr(__entry->sta_addr))
#define STA_NAMED_ASSIGN(s)        memcpy(__entry->sta_addr, (s)->addr, ETH_ALEN)
#define STA_PR_FMT        " sta:%pM"
#define STA_PR_ARG        __entry->sta_addr

#define VIF_ENTRY        __field(enum nl80211_iftype, vif_type) __field(void *, sdata)        \
                        __field(bool, p2p)                                                \
                        __string(vif_name, sdata->name)
#define VIF_ASSIGN        __entry->vif_type = sdata->vif.type; __entry->sdata = sdata;        \
                        __entry->p2p = sdata->vif.p2p;                                        \
                        __assign_str(vif_name)
#define VIF_PR_FMT        " vif:%s(%d%s)"
#define VIF_PR_ARG        __get_str(vif_name), __entry->vif_type, __entry->p2p ? "/p2p" : ""

#define CHANDEF_ENTRY        __field(u32, control_freq)                                        \
                        __field(u32, freq_offset)                                        \
                        __field(u32, chan_width)                                        \
                        __field(u32, center_freq1)                                        \
                        __field(u32, freq1_offset)                                        \
                        __field(u32, center_freq2)
#define CHANDEF_ASSIGN(c)                                                        \
                        __entry->control_freq = (c) ? ((c)->chan ? (c)->chan->center_freq : 0) : 0;        \
                        __entry->freq_offset = (c) ? ((c)->chan ? (c)->chan->freq_offset : 0) : 0;        \
                        __entry->chan_width = (c) ? (c)->width : 0;                        \
                        __entry->center_freq1 = (c) ? (c)->center_freq1 : 0;                \
                        __entry->freq1_offset = (c) ? (c)->freq1_offset : 0;                \
                        __entry->center_freq2 = (c) ? (c)->center_freq2 : 0;
#define CHANDEF_PR_FMT        " chandef(%d.%03d MHz,width:%d,center: %d.%03d/%d MHz)"
#define CHANDEF_PR_ARG        __entry->control_freq, __entry->freq_offset, __entry->chan_width, \
                        __entry->center_freq1, __entry->freq1_offset, __entry->center_freq2

#define MIN_CHANDEF_ENTRY                                                                \
                        __field(u32, min_control_freq)                                        \
                        __field(u32, min_freq_offset)                                        \
                        __field(u32, min_chan_width)                                        \
                        __field(u32, min_center_freq1)                                        \
                        __field(u32, min_freq1_offset)                                        \
                        __field(u32, min_center_freq2)

#define MIN_CHANDEF_ASSIGN(c)                                                                \
                        __entry->min_control_freq = (c)->chan ? (c)->chan->center_freq : 0;        \
                        __entry->min_freq_offset = (c)->chan ? (c)->chan->freq_offset : 0;        \
                        __entry->min_chan_width = (c)->width;                                \
                        __entry->min_center_freq1 = (c)->center_freq1;                        \
                        __entry->min_freq1_offset = (c)->freq1_offset;                        \
                        __entry->min_center_freq2 = (c)->center_freq2;
#define MIN_CHANDEF_PR_FMT        " mindef(%d.%03d MHz,width:%d,center: %d.%03d/%d MHz)"
#define MIN_CHANDEF_PR_ARG        __entry->min_control_freq, __entry->min_freq_offset,        \
                        __entry->min_chan_width,                                        \
                        __entry->min_center_freq1, __entry->min_freq1_offset,                \
                        __entry->min_center_freq2

#define AP_CHANDEF_ENTRY                                                                \
                        __field(u32, ap_control_freq)                                        \
                        __field(u32, ap_freq_offset)                                        \
                        __field(u32, ap_chan_width)                                        \
                        __field(u32, ap_center_freq1)                                        \
                        __field(u32, ap_freq1_offset)                                        \
                        __field(u32, ap_center_freq2)

#define AP_CHANDEF_ASSIGN(c)                                                                \
                        __entry->ap_control_freq = (c)->chan ? (c)->chan->center_freq : 0;\
                        __entry->ap_freq_offset = (c)->chan ? (c)->chan->freq_offset : 0;\
                        __entry->ap_chan_width = (c)->chan ? (c)->width : 0;                \
                        __entry->ap_center_freq1 = (c)->chan ? (c)->center_freq1 : 0;        \
                        __entry->ap_freq1_offset = (c)->chan ? (c)->freq1_offset : 0;        \
                        __entry->ap_center_freq2 = (c)->chan ? (c)->center_freq2 : 0;
#define AP_CHANDEF_PR_FMT        " ap(%d.%03d MHz,width:%d,center: %d.%03d/%d MHz)"
#define AP_CHANDEF_PR_ARG        __entry->ap_control_freq, __entry->ap_freq_offset,        \
                        __entry->ap_chan_width,                                                \
                        __entry->ap_center_freq1, __entry->ap_freq1_offset,                \
                        __entry->ap_center_freq2

#define CHANCTX_ENTRY        CHANDEF_ENTRY                                                        \
                        MIN_CHANDEF_ENTRY                                                \
                        AP_CHANDEF_ENTRY                                                \
                        __field(u8, rx_chains_static)                                        \
                        __field(u8, rx_chains_dynamic)
#define CHANCTX_ASSIGN        CHANDEF_ASSIGN(&ctx->conf.def)                                        \
                        MIN_CHANDEF_ASSIGN(&ctx->conf.min_def)                                \
                        AP_CHANDEF_ASSIGN(&ctx->conf.ap)                                \
                        __entry->rx_chains_static = ctx->conf.rx_chains_static;                \
                        __entry->rx_chains_dynamic = ctx->conf.rx_chains_dynamic
#define CHANCTX_PR_FMT        CHANDEF_PR_FMT MIN_CHANDEF_PR_FMT AP_CHANDEF_PR_FMT " chains:%d/%d"
#define CHANCTX_PR_ARG        CHANDEF_PR_ARG,        MIN_CHANDEF_PR_ARG, AP_CHANDEF_PR_ARG,                \
                        __entry->rx_chains_static, __entry->rx_chains_dynamic

#define KEY_ENTRY        __field(u32, cipher)                                                \
                        __field(u8, hw_key_idx)                                                \
                        __field(u8, flags)                                                \
                        __field(s8, keyidx)
#define KEY_ASSIGN(k)        __entry->cipher = (k)->cipher;                                        \
                        __entry->flags = (k)->flags;                                        \
                        __entry->keyidx = (k)->keyidx;                                        \
                        __entry->hw_key_idx = (k)->hw_key_idx;
#define KEY_PR_FMT        " cipher:0x%x, flags=%#x, keyidx=%d, hw_key_idx=%d"
#define KEY_PR_ARG        __entry->cipher, __entry->flags, __entry->keyidx, __entry->hw_key_idx

#define AMPDU_ACTION_ENTRY        __field(enum ieee80211_ampdu_mlme_action,                \
                                        ieee80211_ampdu_mlme_action)                        \
                                STA_ENTRY                                                \
                                __field(u16, tid)                                        \
                                __field(u16, ssn)                                        \
                                __field(u16, buf_size)                                        \
                                __field(bool, amsdu)                                        \
                                __field(u16, timeout)                                        \
                                __field(u16, action)
#define AMPDU_ACTION_ASSIGN        STA_NAMED_ASSIGN(params->sta);                                \
                                __entry->tid = params->tid;                                \
                                __entry->ssn = params->ssn;                                \
                                __entry->buf_size = params->buf_size;                        \
                                __entry->amsdu = params->amsdu;                                \
                                __entry->timeout = params->timeout;                        \
                                __entry->action = params->action;
#define AMPDU_ACTION_PR_FMT        STA_PR_FMT " tid %d, ssn %d, buf_size %u, amsdu %d, timeout %d action %d"
#define AMPDU_ACTION_PR_ARG        STA_PR_ARG, __entry->tid, __entry->ssn,                        \
                                __entry->buf_size, __entry->amsdu, __entry->timeout,        \
                                __entry->action

/*
 * Tracing for driver callbacks.
 */

DECLARE_EVENT_CLASS(local_only_evt,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local),
        TP_STRUCT__entry(
                LOCAL_ENTRY
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
        ),
        TP_printk(LOCAL_PR_FMT, LOCAL_PR_ARG)
);

DECLARE_EVENT_CLASS(local_sdata_addr_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __array(char, addr, ETH_ALEN)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                memcpy(__entry->addr, sdata->vif.addr, ETH_ALEN);
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " addr:%pM",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->addr
        )
);

DECLARE_EVENT_CLASS(local_u32_evt,
        TP_PROTO(struct ieee80211_local *local, u32 value),
        TP_ARGS(local, value),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, value)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->value = value;
        ),

        TP_printk(
                LOCAL_PR_FMT " value:%d",
                LOCAL_PR_ARG, __entry->value
        )
);

DECLARE_EVENT_CLASS(local_sdata_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG
        )
);

DEFINE_EVENT(local_only_evt, drv_return_void,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(drv_return_int,
        TP_PROTO(struct ieee80211_local *local, int ret),
        TP_ARGS(local, ret),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, ret)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
        ),
        TP_printk(LOCAL_PR_FMT " - %d", LOCAL_PR_ARG, __entry->ret)
);

TRACE_EVENT(drv_return_bool,
        TP_PROTO(struct ieee80211_local *local, bool ret),
        TP_ARGS(local, ret),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, ret)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
        ),
        TP_printk(LOCAL_PR_FMT " - %s", LOCAL_PR_ARG, (__entry->ret) ?
                  "true" : "false")
);

TRACE_EVENT(drv_return_u32,
        TP_PROTO(struct ieee80211_local *local, u32 ret),
        TP_ARGS(local, ret),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, ret)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
        ),
        TP_printk(LOCAL_PR_FMT " - %u", LOCAL_PR_ARG, __entry->ret)
);

TRACE_EVENT(drv_return_u64,
        TP_PROTO(struct ieee80211_local *local, u64 ret),
        TP_ARGS(local, ret),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u64, ret)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
        ),
        TP_printk(LOCAL_PR_FMT " - %llu", LOCAL_PR_ARG, __entry->ret)
);

DEFINE_EVENT(local_only_evt, drv_start,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

DEFINE_EVENT(local_u32_evt, drv_get_et_strings,
             TP_PROTO(struct ieee80211_local *local, u32 sset),
             TP_ARGS(local, sset)
);

DEFINE_EVENT(local_u32_evt, drv_get_et_sset_count,
             TP_PROTO(struct ieee80211_local *local, u32 sset),
             TP_ARGS(local, sset)
);

DEFINE_EVENT(local_only_evt, drv_get_et_stats,
             TP_PROTO(struct ieee80211_local *local),
             TP_ARGS(local)
);

DEFINE_EVENT(local_only_evt, drv_suspend,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

DEFINE_EVENT(local_only_evt, drv_resume,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(drv_set_wakeup,
        TP_PROTO(struct ieee80211_local *local, bool enabled),
        TP_ARGS(local, enabled),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, enabled)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->enabled = enabled;
        ),
        TP_printk(LOCAL_PR_FMT " enabled:%d", LOCAL_PR_ARG, __entry->enabled)
);

TRACE_EVENT(drv_stop,
        TP_PROTO(struct ieee80211_local *local, bool suspend),
        TP_ARGS(local, suspend),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, suspend)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->suspend = suspend;
        ),
        TP_printk(LOCAL_PR_FMT " suspend:%d", LOCAL_PR_ARG, __entry->suspend)
);

DEFINE_EVENT(local_sdata_addr_evt, drv_add_interface,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_change_interface,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 enum nl80211_iftype type, bool p2p),

        TP_ARGS(local, sdata, type, p2p),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, new_type)
                __field(bool, new_p2p)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->new_type = type;
                __entry->new_p2p = p2p;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " new type:%d%s",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->new_type,
                __entry->new_p2p ? "/p2p" : ""
        )
);

DEFINE_EVENT(local_sdata_addr_evt, drv_remove_interface,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_config,
        TP_PROTO(struct ieee80211_local *local,
                 u32 changed),

        TP_ARGS(local, changed),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, changed)
                __field(u32, flags)
                __field(int, power_level)
                __field(int, dynamic_ps_timeout)
                __field(u16, listen_interval)
                __field(u8, long_frame_max_tx_count)
                __field(u8, short_frame_max_tx_count)
                CHANDEF_ENTRY
                __field(int, smps)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->changed = changed;
                __entry->flags = local->hw.conf.flags;
                __entry->power_level = local->hw.conf.power_level;
                __entry->dynamic_ps_timeout = local->hw.conf.dynamic_ps_timeout;
                __entry->listen_interval = local->hw.conf.listen_interval;
                __entry->long_frame_max_tx_count =
                        local->hw.conf.long_frame_max_tx_count;
                __entry->short_frame_max_tx_count =
                        local->hw.conf.short_frame_max_tx_count;
                CHANDEF_ASSIGN(&local->hw.conf.chandef)
                __entry->smps = local->hw.conf.smps_mode;
        ),

        TP_printk(
                LOCAL_PR_FMT " ch:%#x" CHANDEF_PR_FMT,
                LOCAL_PR_ARG, __entry->changed, CHANDEF_PR_ARG
        )
);

TRACE_EVENT(drv_vif_cfg_changed,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u64 changed),

        TP_ARGS(local, sdata, changed),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u64, changed)
                __field(bool, assoc)
                __field(bool, ibss_joined)
                __field(bool, ibss_creator)
                __field(u16, aid)
                __dynamic_array(u32, arp_addr_list,
                                sdata->vif.cfg.arp_addr_cnt > IEEE80211_BSS_ARP_ADDR_LIST_LEN ?
                                        IEEE80211_BSS_ARP_ADDR_LIST_LEN :
                                        sdata->vif.cfg.arp_addr_cnt)
                __field(int, arp_addr_cnt)
                __dynamic_array(u8, ssid, sdata->vif.cfg.ssid_len)
                __field(int, s1g)
                __field(bool, idle)
                __field(bool, ps)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->changed = changed;
                __entry->aid = sdata->vif.cfg.aid;
                __entry->assoc = sdata->vif.cfg.assoc;
                __entry->ibss_joined = sdata->vif.cfg.ibss_joined;
                __entry->ibss_creator = sdata->vif.cfg.ibss_creator;
                __entry->ps = sdata->vif.cfg.ps;

                __entry->arp_addr_cnt = sdata->vif.cfg.arp_addr_cnt;
                memcpy(__get_dynamic_array(arp_addr_list),
                       sdata->vif.cfg.arp_addr_list,
                       sizeof(u32) * (sdata->vif.cfg.arp_addr_cnt > IEEE80211_BSS_ARP_ADDR_LIST_LEN ?
                                        IEEE80211_BSS_ARP_ADDR_LIST_LEN :
                                        sdata->vif.cfg.arp_addr_cnt));
                memcpy(__get_dynamic_array(ssid),
                       sdata->vif.cfg.ssid,
                       sdata->vif.cfg.ssid_len);
                __entry->s1g = sdata->vif.cfg.s1g;
                __entry->idle = sdata->vif.cfg.idle;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " changed:%#llx",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->changed
        )
);

TRACE_EVENT(drv_link_info_changed,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf,
                 u64 changed),

        TP_ARGS(local, sdata, link_conf, changed),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u64, changed)
                __field(int, link_id)
                __field(bool, cts)
                __field(bool, shortpre)
                __field(bool, shortslot)
                __field(bool, enable_beacon)
                __field(u8, dtimper)
                __field(u16, bcnint)
                __field(u16, assoc_cap)
                __field(u64, sync_tsf)
                __field(u32, sync_device_ts)
                __field(u8, sync_dtim_count)
                __field(u32, basic_rates)
                __array(int, mcast_rate, NUM_NL80211_BANDS)
                __field(u16, ht_operation_mode)
                __field(s32, cqm_rssi_thold)
                __field(s32, cqm_rssi_hyst)
                __field(u32, channel_width)
                __field(u32, channel_cfreq1)
                __field(u32, channel_cfreq1_offset)
                __field(bool, qos)
                __field(bool, hidden_ssid)
                __field(int, txpower)
                __field(u8, p2p_oppps_ctwindow)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->changed = changed;
                __entry->link_id = link_conf->link_id;
                __entry->shortpre = link_conf->use_short_preamble;
                __entry->cts = link_conf->use_cts_prot;
                __entry->shortslot = link_conf->use_short_slot;
                __entry->enable_beacon = link_conf->enable_beacon;
                __entry->dtimper = link_conf->dtim_period;
                __entry->bcnint = link_conf->beacon_int;
                __entry->assoc_cap = link_conf->assoc_capability;
                __entry->sync_tsf = link_conf->sync_tsf;
                __entry->sync_device_ts = link_conf->sync_device_ts;
                __entry->sync_dtim_count = link_conf->sync_dtim_count;
                __entry->basic_rates = link_conf->basic_rates;
                memcpy(__entry->mcast_rate, link_conf->mcast_rate,
                       sizeof(__entry->mcast_rate));
                __entry->ht_operation_mode = link_conf->ht_operation_mode;
                __entry->cqm_rssi_thold = link_conf->cqm_rssi_thold;
                __entry->cqm_rssi_hyst = link_conf->cqm_rssi_hyst;
                __entry->channel_width = link_conf->chanreq.oper.width;
                __entry->channel_cfreq1 = link_conf->chanreq.oper.center_freq1;
                __entry->channel_cfreq1_offset = link_conf->chanreq.oper.freq1_offset;
                __entry->qos = link_conf->qos;
                __entry->hidden_ssid = link_conf->hidden_ssid;
                __entry->txpower = link_conf->txpower;
                __entry->p2p_oppps_ctwindow = link_conf->p2p_noa_attr.oppps_ctwindow;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " link_id:%d, changed:%#llx",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id,
                __entry->changed
        )
);

TRACE_EVENT(drv_prepare_multicast,
        TP_PROTO(struct ieee80211_local *local, int mc_count),

        TP_ARGS(local, mc_count),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, mc_count)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->mc_count = mc_count;
        ),

        TP_printk(
                LOCAL_PR_FMT " prepare mc (%d)",
                LOCAL_PR_ARG, __entry->mc_count
        )
);

TRACE_EVENT(drv_configure_filter,
        TP_PROTO(struct ieee80211_local *local,
                 unsigned int changed_flags,
                 unsigned int *total_flags,
                 u64 multicast),

        TP_ARGS(local, changed_flags, total_flags, multicast),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(unsigned int, changed)
                __field(unsigned int, total)
                __field(u64, multicast)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->changed = changed_flags;
                __entry->total = *total_flags;
                __entry->multicast = multicast;
        ),

        TP_printk(
                LOCAL_PR_FMT " changed:%#x total:%#x",
                LOCAL_PR_ARG, __entry->changed, __entry->total
        )
);

TRACE_EVENT(drv_config_iface_filter,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 unsigned int filter_flags,
                 unsigned int changed_flags),

        TP_ARGS(local, sdata, filter_flags, changed_flags),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(unsigned int, filter_flags)
                __field(unsigned int, changed_flags)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->filter_flags = filter_flags;
                __entry->changed_flags = changed_flags;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT
                " filter_flags: %#x changed_flags: %#x",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->filter_flags,
                __entry->changed_flags
        )
);

TRACE_EVENT(drv_set_tim,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta, bool set),

        TP_ARGS(local, sta, set),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(bool, set)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->set = set;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT " set:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->set
        )
);

TRACE_EVENT(drv_set_key,
        TP_PROTO(struct ieee80211_local *local,
                 enum set_key_cmd cmd, struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta,
                 struct ieee80211_key_conf *key),

        TP_ARGS(local, cmd, sdata, sta, key),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u32, cmd)
                KEY_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->cmd = cmd;
                KEY_ASSIGN(key);
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " cmd: %d" KEY_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->cmd, KEY_PR_ARG
        )
);

TRACE_EVENT(drv_update_tkip_key,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_key_conf *conf,
                 struct ieee80211_sta *sta, u32 iv32),

        TP_ARGS(local, sdata, conf, sta, iv32),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u32, iv32)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->iv32 = iv32;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT STA_PR_FMT " iv32:%#x",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->iv32
        )
);

DEFINE_EVENT(local_sdata_evt, drv_hw_scan,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_sdata_evt, drv_cancel_hw_scan,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_sdata_evt, drv_sched_scan_start,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_sdata_evt, drv_sched_scan_stop,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_sw_scan_start,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 const u8 *mac_addr),

        TP_ARGS(local, sdata, mac_addr),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __array(char, mac_addr, ETH_ALEN)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                memcpy(__entry->mac_addr, mac_addr, ETH_ALEN);
        ),

        TP_printk(LOCAL_PR_FMT ", " VIF_PR_FMT ", addr:%pM",
                  LOCAL_PR_ARG, VIF_PR_ARG, __entry->mac_addr)
);

DEFINE_EVENT(local_sdata_evt, drv_sw_scan_complete,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_get_stats,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_low_level_stats *stats,
                 int ret),

        TP_ARGS(local, stats, ret),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, ret)
                __field(unsigned int, ackfail)
                __field(unsigned int, rtsfail)
                __field(unsigned int, fcserr)
                __field(unsigned int, rtssucc)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
                __entry->ackfail = stats->dot11ACKFailureCount;
                __entry->rtsfail = stats->dot11RTSFailureCount;
                __entry->fcserr = stats->dot11FCSErrorCount;
                __entry->rtssucc = stats->dot11RTSSuccessCount;
        ),

        TP_printk(
                LOCAL_PR_FMT " ret:%d",
                LOCAL_PR_ARG, __entry->ret
        )
);

TRACE_EVENT(drv_get_key_seq,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_key_conf *key),

        TP_ARGS(local, key),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                KEY_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                KEY_ASSIGN(key);
        ),

        TP_printk(
                LOCAL_PR_FMT KEY_PR_FMT,
                LOCAL_PR_ARG, KEY_PR_ARG
        )
);

DEFINE_EVENT(local_u32_evt, drv_set_frag_threshold,
        TP_PROTO(struct ieee80211_local *local, u32 value),
        TP_ARGS(local, value)
);

DEFINE_EVENT(local_u32_evt, drv_set_rts_threshold,
        TP_PROTO(struct ieee80211_local *local, u32 value),
        TP_ARGS(local, value)
);

TRACE_EVENT(drv_set_coverage_class,
        TP_PROTO(struct ieee80211_local *local, s16 value),

        TP_ARGS(local, value),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(s16, value)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->value = value;
        ),

        TP_printk(
                LOCAL_PR_FMT " value:%d",
                LOCAL_PR_ARG, __entry->value
        )
);

TRACE_EVENT(drv_sta_notify,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 enum sta_notify_cmd cmd,
                 struct ieee80211_sta *sta),

        TP_ARGS(local, sdata, cmd, sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u32, cmd)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->cmd = cmd;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " cmd:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->cmd
        )
);

TRACE_EVENT(drv_sta_state,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta,
                 enum ieee80211_sta_state old_state,
                 enum ieee80211_sta_state new_state),

        TP_ARGS(local, sdata, sta, old_state, new_state),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u32, old_state)
                __field(u32, new_state)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->old_state = old_state;
                __entry->new_state = new_state;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " state: %d->%d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG,
                __entry->old_state, __entry->new_state
        )
);

TRACE_EVENT(drv_sta_set_txpwr,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),

        TP_ARGS(local, sdata, sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(s16, txpwr)
                __field(u8, type)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->txpwr = sta->deflink.txpwr.power;
                __entry->type = sta->deflink.txpwr.type;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " txpwr: %d type %d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG,
                __entry->txpwr,  __entry->type
        )
);

TRACE_EVENT(drv_link_sta_rc_update,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_link_sta *link_sta,
                 u32 changed),

        TP_ARGS(local, sdata, link_sta, changed),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u32, changed)
                __field(u32, link_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_NAMED_ASSIGN(link_sta->sta);
                __entry->changed = changed;
                __entry->link_id = link_sta->link_id;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " (link %d) changed: 0x%x",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->link_id,
                __entry->changed
        )
);

DECLARE_EVENT_CLASS(sta_event,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),

        TP_ARGS(local, sdata, sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG
        )
);

DEFINE_EVENT(sta_event, drv_sta_statistics,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DEFINE_EVENT(sta_event, drv_sta_add,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DEFINE_EVENT(sta_event, drv_sta_remove,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DEFINE_EVENT(sta_event, drv_sta_pre_rcu_remove,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DEFINE_EVENT(sta_event, drv_sync_rx_queues,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DEFINE_EVENT(sta_event, drv_sta_rate_tbl_update,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

TRACE_EVENT(drv_conf_tx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 unsigned int link_id,
                 u16 ac, const struct ieee80211_tx_queue_params *params),

        TP_ARGS(local, sdata, link_id, ac, params),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(unsigned int, link_id)
                __field(u16, ac)
                __field(u16, txop)
                __field(u16, cw_min)
                __field(u16, cw_max)
                __field(u8, aifs)
                __field(bool, uapsd)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->link_id = link_id;
                __entry->ac = ac;
                __entry->txop = params->txop;
                __entry->cw_max = params->cw_max;
                __entry->cw_min = params->cw_min;
                __entry->aifs = params->aifs;
                __entry->uapsd = params->uapsd;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  " link_id: %d, AC:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id, __entry->ac
        )
);

DEFINE_EVENT(local_sdata_evt, drv_get_tsf,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_set_tsf,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u64 tsf),

        TP_ARGS(local, sdata, tsf),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u64, tsf)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->tsf = tsf;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  " tsf:%llu",
                LOCAL_PR_ARG, VIF_PR_ARG, (unsigned long long)__entry->tsf
        )
);

TRACE_EVENT(drv_offset_tsf,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 s64 offset),

        TP_ARGS(local, sdata, offset),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(s64, tsf_offset)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->tsf_offset = offset;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  " tsf offset:%lld",
                LOCAL_PR_ARG, VIF_PR_ARG,
                (unsigned long long)__entry->tsf_offset
        )
);

DEFINE_EVENT(local_sdata_evt, drv_reset_tsf,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_only_evt, drv_tx_last_beacon,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(drv_ampdu_action,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_ampdu_params *params),

        TP_ARGS(local, sdata, params),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                AMPDU_ACTION_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                AMPDU_ACTION_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT AMPDU_ACTION_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, AMPDU_ACTION_PR_ARG
        )
);

TRACE_EVENT(drv_get_survey,
        TP_PROTO(struct ieee80211_local *local, int _idx,
                 struct survey_info *survey),

        TP_ARGS(local, _idx, survey),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, idx)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->idx = _idx;
        ),

        TP_printk(
                LOCAL_PR_FMT " idx:%d",
                LOCAL_PR_ARG, __entry->idx
        )
);

TRACE_EVENT(drv_flush,
        TP_PROTO(struct ieee80211_local *local,
                 u32 queues, bool drop),

        TP_ARGS(local, queues, drop),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, drop)
                __field(u32, queues)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->drop = drop;
                __entry->queues = queues;
        ),

        TP_printk(
                LOCAL_PR_FMT " queues:0x%x drop:%d",
                LOCAL_PR_ARG, __entry->queues, __entry->drop
        )
);

DEFINE_EVENT(sta_event, drv_flush_sta,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

DECLARE_EVENT_CLASS(chanswitch_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_channel_switch *ch_switch),

        TP_ARGS(local, sdata, ch_switch),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                CHANDEF_ENTRY
                __field(u64, timestamp)
                __field(u32, device_timestamp)
                __field(bool, block_tx)
                __field(u8, count)
                __field(u8, link_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                CHANDEF_ASSIGN(&ch_switch->chandef)
                __entry->timestamp = ch_switch->timestamp;
                __entry->device_timestamp = ch_switch->device_timestamp;
                __entry->block_tx = ch_switch->block_tx;
                __entry->count = ch_switch->count;
                __entry->link_id = ch_switch->link_id;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT CHANDEF_PR_FMT  " count:%d block_tx:%d timestamp:%llu device_ts:%u link_id:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, CHANDEF_PR_ARG, __entry->count,
                __entry->block_tx, __entry->timestamp,
                __entry->device_timestamp, __entry->link_id
        )
);

DEFINE_EVENT(chanswitch_evt, drv_channel_switch,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_channel_switch *ch_switch),
        TP_ARGS(local, sdata, ch_switch)
);

TRACE_EVENT(drv_set_antenna,
        TP_PROTO(struct ieee80211_local *local, u32 tx_ant, u32 rx_ant, int ret),

        TP_ARGS(local, tx_ant, rx_ant, ret),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, tx_ant)
                __field(u32, rx_ant)
                __field(int, ret)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->tx_ant = tx_ant;
                __entry->rx_ant = rx_ant;
                __entry->ret = ret;
        ),

        TP_printk(
                LOCAL_PR_FMT " tx_ant:%d rx_ant:%d ret:%d",
                LOCAL_PR_ARG, __entry->tx_ant, __entry->rx_ant, __entry->ret
        )
);

TRACE_EVENT(drv_get_antenna,
        TP_PROTO(struct ieee80211_local *local, u32 tx_ant, u32 rx_ant, int ret),

        TP_ARGS(local, tx_ant, rx_ant, ret),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, tx_ant)
                __field(u32, rx_ant)
                __field(int, ret)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->tx_ant = tx_ant;
                __entry->rx_ant = rx_ant;
                __entry->ret = ret;
        ),

        TP_printk(
                LOCAL_PR_FMT " tx_ant:%d rx_ant:%d ret:%d",
                LOCAL_PR_ARG, __entry->tx_ant, __entry->rx_ant, __entry->ret
        )
);

TRACE_EVENT(drv_remain_on_channel,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_channel *chan,
                 unsigned int duration,
                 enum ieee80211_roc_type type),

        TP_ARGS(local, sdata, chan, duration, type),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(int, center_freq)
                __field(int, freq_offset)
                __field(unsigned int, duration)
                __field(u32, type)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->center_freq = chan->center_freq;
                __entry->freq_offset = chan->freq_offset;
                __entry->duration = duration;
                __entry->type = type;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " freq:%d.%03dMHz duration:%dms type=%d",
                LOCAL_PR_ARG, VIF_PR_ARG,
                __entry->center_freq, __entry->freq_offset,
                __entry->duration, __entry->type
        )
);

DEFINE_EVENT(local_sdata_evt, drv_cancel_remain_on_channel,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_set_ringparam,
        TP_PROTO(struct ieee80211_local *local, u32 tx, u32 rx),

        TP_ARGS(local, tx, rx),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, tx)
                __field(u32, rx)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->tx = tx;
                __entry->rx = rx;
        ),

        TP_printk(
                LOCAL_PR_FMT " tx:%d rx %d",
                LOCAL_PR_ARG, __entry->tx, __entry->rx
        )
);

TRACE_EVENT(drv_get_ringparam,
        TP_PROTO(struct ieee80211_local *local, u32 *tx, u32 *tx_max,
                 u32 *rx, u32 *rx_max),

        TP_ARGS(local, tx, tx_max, rx, rx_max),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u32, tx)
                __field(u32, tx_max)
                __field(u32, rx)
                __field(u32, rx_max)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->tx = *tx;
                __entry->tx_max = *tx_max;
                __entry->rx = *rx;
                __entry->rx_max = *rx_max;
        ),

        TP_printk(
                LOCAL_PR_FMT " tx:%d tx_max %d rx %d rx_max %d",
                LOCAL_PR_ARG,
                __entry->tx, __entry->tx_max, __entry->rx, __entry->rx_max
        )
);

DEFINE_EVENT(local_only_evt, drv_tx_frames_pending,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

DEFINE_EVENT(local_only_evt, drv_offchannel_tx_cancel_wait,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(drv_set_bitrate_mask,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 const struct cfg80211_bitrate_mask *mask),

        TP_ARGS(local, sdata, mask),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, legacy_2g)
                __field(u32, legacy_5g)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->legacy_2g = mask->control[NL80211_BAND_2GHZ].legacy;
                __entry->legacy_5g = mask->control[NL80211_BAND_5GHZ].legacy;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " 2G Mask:0x%x 5G Mask:0x%x",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->legacy_2g, __entry->legacy_5g
        )
);

TRACE_EVENT(drv_set_rekey_data,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct cfg80211_gtk_rekey_data *data),

        TP_ARGS(local, sdata, data),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __array(u8, kek, NL80211_KEK_LEN)
                __array(u8, kck, NL80211_KCK_LEN)
                __array(u8, replay_ctr, NL80211_REPLAY_CTR_LEN)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                memcpy(__entry->kek, data->kek, NL80211_KEK_LEN);
                memcpy(__entry->kck, data->kck, NL80211_KCK_LEN);
                memcpy(__entry->replay_ctr, data->replay_ctr,
                       NL80211_REPLAY_CTR_LEN);
        ),

        TP_printk(LOCAL_PR_FMT VIF_PR_FMT,
                  LOCAL_PR_ARG, VIF_PR_ARG)
);

TRACE_EVENT(drv_event_callback,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 const struct ieee80211_event *_event),

        TP_ARGS(local, sdata, _event),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, type)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->type = _event->type;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " event:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->type
        )
);

DECLARE_EVENT_CLASS(release_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 u16 tids, int num_frames,
                 enum ieee80211_frame_release_type reason,
                 bool more_data),

        TP_ARGS(local, sta, tids, num_frames, reason, more_data),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(u16, tids)
                __field(int, num_frames)
                __field(int, reason)
                __field(bool, more_data)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->tids = tids;
                __entry->num_frames = num_frames;
                __entry->reason = reason;
                __entry->more_data = more_data;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT
                " TIDs:0x%.4x frames:%d reason:%d more:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->tids, __entry->num_frames,
                __entry->reason, __entry->more_data
        )
);

DEFINE_EVENT(release_evt, drv_release_buffered_frames,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 u16 tids, int num_frames,
                 enum ieee80211_frame_release_type reason,
                 bool more_data),

        TP_ARGS(local, sta, tids, num_frames, reason, more_data)
);

DEFINE_EVENT(release_evt, drv_allow_buffered_frames,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 u16 tids, int num_frames,
                 enum ieee80211_frame_release_type reason,
                 bool more_data),

        TP_ARGS(local, sta, tids, num_frames, reason, more_data)
);

DECLARE_EVENT_CLASS(mgd_prepare_complete_tx_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u16 duration, u16 subtype, bool success),

        TP_ARGS(local, sdata, duration, subtype, success),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, duration)
                __field(u16, subtype)
                __field(u8, success)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->duration = duration;
                __entry->subtype = subtype;
                __entry->success = success;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " duration: %u, subtype:0x%x, success:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->duration,
                __entry->subtype, __entry->success
        )
);

DEFINE_EVENT(mgd_prepare_complete_tx_evt, drv_mgd_prepare_tx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u16 duration, u16 subtype, bool success),

        TP_ARGS(local, sdata, duration, subtype, success)
);

DEFINE_EVENT(mgd_prepare_complete_tx_evt, drv_mgd_complete_tx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u16 duration, u16 subtype, bool success),

        TP_ARGS(local, sdata, duration, subtype, success)
);

DEFINE_EVENT(local_sdata_evt, drv_mgd_protect_tdls_discover,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),

        TP_ARGS(local, sdata)
);

DECLARE_EVENT_CLASS(local_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_chanctx *ctx),

        TP_ARGS(local, ctx),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                CHANCTX_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                CHANCTX_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT CHANCTX_PR_FMT,
                LOCAL_PR_ARG, CHANCTX_PR_ARG
        )
);

DEFINE_EVENT(local_chanctx, drv_add_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_chanctx *ctx),
        TP_ARGS(local, ctx)
);

DEFINE_EVENT(local_chanctx, drv_remove_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_chanctx *ctx),
        TP_ARGS(local, ctx)
);

TRACE_EVENT(drv_change_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_chanctx *ctx,
                 u32 changed),

        TP_ARGS(local, ctx, changed),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                CHANCTX_ENTRY
                __field(u32, changed)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                CHANCTX_ASSIGN;
                __entry->changed = changed;
        ),

        TP_printk(
                LOCAL_PR_FMT CHANCTX_PR_FMT " changed:%#x",
                LOCAL_PR_ARG, CHANCTX_PR_ARG, __entry->changed
        )
);

#if !defined(__TRACE_VIF_ENTRY)
#define __TRACE_VIF_ENTRY
struct trace_vif_entry {
        enum nl80211_iftype vif_type;
        bool p2p;
        char vif_name[IFNAMSIZ];
} __packed;

struct trace_chandef_entry {
        u32 control_freq;
        u32 freq_offset;
        u32 chan_width;
        u32 center_freq1;
        u32 freq1_offset;
        u32 center_freq2;
} __packed;

struct trace_switch_entry {
        struct trace_vif_entry vif;
        unsigned int link_id;
        struct trace_chandef_entry old_chandef;
        struct trace_chandef_entry new_chandef;
} __packed;

#define SWITCH_ENTRY_ASSIGN(to, from) local_vifs[i].to = vifs[i].from
#endif

TRACE_EVENT(drv_switch_vif_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_vif_chanctx_switch *vifs,
                 int n_vifs, enum ieee80211_chanctx_switch_mode mode),
            TP_ARGS(local, vifs, n_vifs, mode),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, n_vifs)
                __field(u32, mode)
                __dynamic_array(u8, vifs,
                                sizeof(struct trace_switch_entry) * n_vifs)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->n_vifs = n_vifs;
                __entry->mode = mode;
                {
                        struct trace_switch_entry *local_vifs =
                                __get_dynamic_array(vifs);
                        int i;

                        for (i = 0; i < n_vifs; i++) {
                                struct ieee80211_sub_if_data *sdata;

                                sdata = container_of(vifs[i].vif,
                                                struct ieee80211_sub_if_data,
                                                vif);

                                SWITCH_ENTRY_ASSIGN(vif.vif_type, vif->type);
                                SWITCH_ENTRY_ASSIGN(vif.p2p, vif->p2p);
                                SWITCH_ENTRY_ASSIGN(link_id, link_conf->link_id);
                                strncpy(local_vifs[i].vif.vif_name,
                                        sdata->name,
                                        sizeof(local_vifs[i].vif.vif_name));
                                SWITCH_ENTRY_ASSIGN(old_chandef.control_freq,
                                                old_ctx->def.chan->center_freq);
                                SWITCH_ENTRY_ASSIGN(old_chandef.freq_offset,
                                                old_ctx->def.chan->freq_offset);
                                SWITCH_ENTRY_ASSIGN(old_chandef.chan_width,
                                                    old_ctx->def.width);
                                SWITCH_ENTRY_ASSIGN(old_chandef.center_freq1,
                                                    old_ctx->def.center_freq1);
                                SWITCH_ENTRY_ASSIGN(old_chandef.freq1_offset,
                                                    old_ctx->def.freq1_offset);
                                SWITCH_ENTRY_ASSIGN(old_chandef.center_freq2,
                                                    old_ctx->def.center_freq2);
                                SWITCH_ENTRY_ASSIGN(new_chandef.control_freq,
                                                new_ctx->def.chan->center_freq);
                                SWITCH_ENTRY_ASSIGN(new_chandef.freq_offset,
                                                new_ctx->def.chan->freq_offset);
                                SWITCH_ENTRY_ASSIGN(new_chandef.chan_width,
                                                    new_ctx->def.width);
                                SWITCH_ENTRY_ASSIGN(new_chandef.center_freq1,
                                                    new_ctx->def.center_freq1);
                                SWITCH_ENTRY_ASSIGN(new_chandef.freq1_offset,
                                                    new_ctx->def.freq1_offset);
                                SWITCH_ENTRY_ASSIGN(new_chandef.center_freq2,
                                                    new_ctx->def.center_freq2);
                        }
                }
        ),

        TP_printk(
                LOCAL_PR_FMT " n_vifs:%d mode:%d",
                LOCAL_PR_ARG, __entry->n_vifs, __entry->mode
        )
);

DECLARE_EVENT_CLASS(local_sdata_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf,
                 struct ieee80211_chanctx *ctx),

        TP_ARGS(local, sdata, link_conf, ctx),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                CHANCTX_ENTRY
                __field(unsigned int, link_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                CHANCTX_ASSIGN;
                __entry->link_id = link_conf->link_id;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " link_id:%d" CHANCTX_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id, CHANCTX_PR_ARG
        )
);

DEFINE_EVENT(local_sdata_chanctx, drv_assign_vif_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf,
                 struct ieee80211_chanctx *ctx),
        TP_ARGS(local, sdata, link_conf, ctx)
);

DEFINE_EVENT(local_sdata_chanctx, drv_unassign_vif_chanctx,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf,
                 struct ieee80211_chanctx *ctx),
        TP_ARGS(local, sdata, link_conf, ctx)
);

TRACE_EVENT(drv_start_ap,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf),

        TP_ARGS(local, sdata, link_conf),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, link_id)
                __field(u8, dtimper)
                __field(u16, bcnint)
                __dynamic_array(u8, ssid, sdata->vif.cfg.ssid_len)
                __field(bool, hidden_ssid)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->link_id = link_conf->link_id;
                __entry->dtimper = link_conf->dtim_period;
                __entry->bcnint = link_conf->beacon_int;
                __entry->hidden_ssid = link_conf->hidden_ssid;
                memcpy(__get_dynamic_array(ssid),
                       sdata->vif.cfg.ssid,
                       sdata->vif.cfg.ssid_len);
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " link id %u",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id
        )
);

TRACE_EVENT(drv_stop_ap,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *link_conf),

        TP_ARGS(local, sdata, link_conf),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u32, link_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->link_id = link_conf->link_id;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT " link id %u",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id
        )
);

TRACE_EVENT(drv_reconfig_complete,
        TP_PROTO(struct ieee80211_local *local,
                 enum ieee80211_reconfig_type reconfig_type),
        TP_ARGS(local, reconfig_type),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u8, reconfig_type)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->reconfig_type = reconfig_type;
        ),

        TP_printk(
                LOCAL_PR_FMT  " reconfig_type:%d",
                LOCAL_PR_ARG, __entry->reconfig_type
        )

);

#if IS_ENABLED(CONFIG_IPV6)
DEFINE_EVENT(local_sdata_evt, drv_ipv6_addr_change,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);
#endif

TRACE_EVENT(drv_join_ibss,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_bss_conf *info),

        TP_ARGS(local, sdata, info),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, dtimper)
                __field(u16, bcnint)
                __dynamic_array(u8, ssid, sdata->vif.cfg.ssid_len)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->dtimper = info->dtim_period;
                __entry->bcnint = info->beacon_int;
                memcpy(__get_dynamic_array(ssid),
                       sdata->vif.cfg.ssid,
                       sdata->vif.cfg.ssid_len);
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG
        )
);

DEFINE_EVENT(local_sdata_evt, drv_leave_ibss,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_get_expected_throughput,
        TP_PROTO(struct ieee80211_sta *sta),

        TP_ARGS(sta),

        TP_STRUCT__entry(
                STA_ENTRY
        ),

        TP_fast_assign(
                STA_ASSIGN;
        ),

        TP_printk(
                STA_PR_FMT, STA_PR_ARG
        )
);

TRACE_EVENT(drv_start_nan,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct cfg80211_nan_conf *conf),

        TP_ARGS(local, sdata, conf),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, master_pref)
                __field(u8, bands)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->master_pref = conf->master_pref;
                __entry->bands = conf->bands;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT
                ", master preference: %u, bands: 0x%0x",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->master_pref,
                __entry->bands
        )
);

TRACE_EVENT(drv_stop_nan,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),

        TP_ARGS(local, sdata),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG
        )
);

TRACE_EVENT(drv_nan_change_conf,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct cfg80211_nan_conf *conf,
                 u32 changes),

        TP_ARGS(local, sdata, conf, changes),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, master_pref)
                __field(u8, bands)
                __field(u32, changes)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->master_pref = conf->master_pref;
                __entry->bands = conf->bands;
                __entry->changes = changes;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT
                ", master preference: %u, bands: 0x%0x, changes: 0x%x",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->master_pref,
                __entry->bands, __entry->changes
        )
);

TRACE_EVENT(drv_add_nan_func,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 const struct cfg80211_nan_func *func),

        TP_ARGS(local, sdata, func),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, type)
                __field(u8, inst_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->type = func->type;
                __entry->inst_id = func->instance_id;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT
                ", type: %u, inst_id: %u",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->type, __entry->inst_id
        )
);

TRACE_EVENT(drv_del_nan_func,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u8 instance_id),

        TP_ARGS(local, sdata, instance_id),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, instance_id)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->instance_id = instance_id;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT
                ", instance_id: %u",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->instance_id
        )
);

DEFINE_EVENT(local_sdata_evt, drv_start_pmsr,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_sdata_evt, drv_abort_pmsr,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(drv_set_default_unicast_key,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 int key_idx),

        TP_ARGS(local, sdata, key_idx),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(int, key_idx)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->key_idx = key_idx;
        ),

        TP_printk(LOCAL_PR_FMT VIF_PR_FMT " key_idx:%d",
                  LOCAL_PR_ARG, VIF_PR_ARG, __entry->key_idx)
);

TRACE_EVENT(drv_channel_switch_beacon,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct cfg80211_chan_def *chandef),

        TP_ARGS(local, sdata, chandef),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                CHANDEF_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                CHANDEF_ASSIGN(chandef);
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " channel switch to " CHANDEF_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, CHANDEF_PR_ARG
        )
);

DEFINE_EVENT(chanswitch_evt, drv_pre_channel_switch,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_channel_switch *ch_switch),
        TP_ARGS(local, sdata, ch_switch)
);

DEFINE_EVENT(local_sdata_evt, drv_post_channel_switch,
             TP_PROTO(struct ieee80211_local *local,
                      struct ieee80211_sub_if_data *sdata),
             TP_ARGS(local, sdata)
);

DEFINE_EVENT(local_sdata_evt, drv_abort_channel_switch,
             TP_PROTO(struct ieee80211_local *local,
                      struct ieee80211_sub_if_data *sdata),
             TP_ARGS(local, sdata)
);

DEFINE_EVENT(chanswitch_evt, drv_channel_switch_rx_beacon,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_channel_switch *ch_switch),
        TP_ARGS(local, sdata, ch_switch)
);

TRACE_EVENT(drv_get_txpower,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 int dbm, int ret),

        TP_ARGS(local, sdata, dbm, ret),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(int, dbm)
                __field(int, ret)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->dbm = dbm;
                __entry->ret = ret;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " dbm:%d ret:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->dbm, __entry->ret
        )
);

TRACE_EVENT(drv_tdls_channel_switch,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta, u8 oper_class,
                 struct cfg80211_chan_def *chandef),

        TP_ARGS(local, sdata, sta, oper_class, chandef),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u8, oper_class)
                CHANDEF_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->oper_class = oper_class;
                CHANDEF_ASSIGN(chandef)
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " tdls channel switch to"
                CHANDEF_PR_FMT  " oper_class:%d " STA_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, CHANDEF_PR_ARG, __entry->oper_class,
                STA_PR_ARG
        )
);

TRACE_EVENT(drv_tdls_cancel_channel_switch,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),

        TP_ARGS(local, sdata, sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT
                " tdls cancel channel switch with " STA_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG
        )
);

TRACE_EVENT(drv_tdls_recv_channel_switch,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_tdls_ch_sw_params *params),

        TP_ARGS(local, sdata, params),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, action_code)
                STA_ENTRY
                CHANDEF_ENTRY
                __field(u32, status)
                __field(bool, peer_initiator)
                __field(u32, timestamp)
                __field(u16, switch_time)
                __field(u16, switch_timeout)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_NAMED_ASSIGN(params->sta);
                CHANDEF_ASSIGN(params->chandef)
                __entry->peer_initiator = params->sta->tdls_initiator;
                __entry->action_code = params->action_code;
                __entry->status = params->status;
                __entry->timestamp = params->timestamp;
                __entry->switch_time = params->switch_time;
                __entry->switch_timeout = params->switch_timeout;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " received tdls channel switch packet"
                " action:%d status:%d time:%d switch time:%d switch"
                " timeout:%d initiator: %d chan:" CHANDEF_PR_FMT STA_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->action_code, __entry->status,
                __entry->timestamp, __entry->switch_time,
                __entry->switch_timeout, __entry->peer_initiator,
                CHANDEF_PR_ARG, STA_PR_ARG
        )
);

TRACE_EVENT(drv_wake_tx_queue,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct txq_info *txq),

        TP_ARGS(local, sdata, txq),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u8, ac)
                __field(u8, tid)
        ),

        TP_fast_assign(
                struct ieee80211_sta *sta = txq->txq.sta;

                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->ac = txq->txq.ac;
                __entry->tid = txq->txq.tid;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " ac:%d tid:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->ac, __entry->tid
        )
);

TRACE_EVENT(drv_get_ftm_responder_stats,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct cfg80211_ftm_responder_stats *ftm_stats),

        TP_ARGS(local, sdata, ftm_stats),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT,
                LOCAL_PR_ARG, VIF_PR_ARG
        )
);

DEFINE_EVENT(local_sdata_addr_evt, drv_update_vif_offload,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

DECLARE_EVENT_CLASS(sta_flag_evt,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta, bool enabled),

        TP_ARGS(local, sdata, sta, enabled),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(bool, enabled)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->enabled = enabled;
        ),

        TP_printk(
                LOCAL_PR_FMT  VIF_PR_FMT  STA_PR_FMT " enabled:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG, __entry->enabled
        )
);

DEFINE_EVENT(sta_flag_evt, drv_sta_set_4addr,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta, bool enabled),

        TP_ARGS(local, sdata, sta, enabled)
);

DEFINE_EVENT(sta_flag_evt, drv_sta_set_decap_offload,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta, bool enabled),

        TP_ARGS(local, sdata, sta, enabled)
);

TRACE_EVENT(drv_add_twt_setup,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 struct ieee80211_twt_setup *twt,
                 struct ieee80211_twt_params *twt_agrt),

        TP_ARGS(local, sta, twt, twt_agrt),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(u8, dialog_token)
                __field(u8, control)
                __field(__le16, req_type)
                __field(__le64, twt)
                __field(u8, duration)
                __field(__le16, mantissa)
                __field(u8, channel)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->dialog_token = twt->dialog_token;
                __entry->control = twt->control;
                __entry->req_type = twt_agrt->req_type;
                __entry->twt = twt_agrt->twt;
                __entry->duration = twt_agrt->min_twt_dur;
                __entry->mantissa = twt_agrt->mantissa;
                __entry->channel = twt_agrt->channel;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT
                " token:%d control:0x%02x req_type:0x%04x"
                " twt:%llu duration:%d mantissa:%d channel:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->dialog_token,
                __entry->control, le16_to_cpu(__entry->req_type),
                le64_to_cpu(__entry->twt), __entry->duration,
                le16_to_cpu(__entry->mantissa), __entry->channel
        )
);

TRACE_EVENT(drv_twt_teardown_request,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta, u8 flowid),

        TP_ARGS(local, sta, flowid),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(u8, flowid)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->flowid = flowid;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT " flowid:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->flowid
        )
);

DEFINE_EVENT(sta_event, drv_net_fill_forward_path,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta),
        TP_ARGS(local, sdata, sta)
);

TRACE_EVENT(drv_net_setup_tc,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u8 type),

        TP_ARGS(local, sdata, type),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u8, type)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->type = type;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " type:%d\n",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->type
        )
);

TRACE_EVENT(drv_can_activate_links,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u16 active_links),

        TP_ARGS(local, sdata, active_links),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u16, active_links)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->active_links = active_links;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " requested active_links:0x%04x\n",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->active_links
        )
);

TRACE_EVENT(drv_change_vif_links,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u16 old_links, u16 new_links),

        TP_ARGS(local, sdata, old_links, new_links),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(u16, old_links)
                __field(u16, new_links)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->old_links = old_links;
                __entry->new_links = new_links;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT " old_links:0x%04x, new_links:0x%04x\n",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->old_links, __entry->new_links
        )
);

TRACE_EVENT(drv_change_sta_links,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_sta *sta,
                 u16 old_links, u16 new_links),

        TP_ARGS(local, sdata, sta, old_links, new_links),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                STA_ENTRY
                __field(u16, old_links)
                __field(u16, new_links)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                STA_ASSIGN;
                __entry->old_links = old_links;
                __entry->new_links = new_links;
        ),

        TP_printk(
                LOCAL_PR_FMT VIF_PR_FMT STA_PR_FMT " old_links:0x%04x, new_links:0x%04x\n",
                LOCAL_PR_ARG, VIF_PR_ARG, STA_PR_ARG,
                __entry->old_links, __entry->new_links
        )
);

/*
 * Tracing for API calls that drivers call.
 */

TRACE_EVENT(api_start_tx_ba_session,
        TP_PROTO(struct ieee80211_sta *sta, u16 tid),

        TP_ARGS(sta, tid),

        TP_STRUCT__entry(
                STA_ENTRY
                __field(u16, tid)
        ),

        TP_fast_assign(
                STA_ASSIGN;
                __entry->tid = tid;
        ),

        TP_printk(
                STA_PR_FMT " tid:%d",
                STA_PR_ARG, __entry->tid
        )
);

TRACE_EVENT(api_start_tx_ba_cb,
        TP_PROTO(struct ieee80211_sub_if_data *sdata, const u8 *ra, u16 tid),

        TP_ARGS(sdata, ra, tid),

        TP_STRUCT__entry(
                VIF_ENTRY
                __array(u8, ra, ETH_ALEN)
                __field(u16, tid)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                memcpy(__entry->ra, ra, ETH_ALEN);
                __entry->tid = tid;
        ),

        TP_printk(
                VIF_PR_FMT " ra:%pM tid:%d",
                VIF_PR_ARG, __entry->ra, __entry->tid
        )
);

TRACE_EVENT(api_stop_tx_ba_session,
        TP_PROTO(struct ieee80211_sta *sta, u16 tid),

        TP_ARGS(sta, tid),

        TP_STRUCT__entry(
                STA_ENTRY
                __field(u16, tid)
        ),

        TP_fast_assign(
                STA_ASSIGN;
                __entry->tid = tid;
        ),

        TP_printk(
                STA_PR_FMT " tid:%d",
                STA_PR_ARG, __entry->tid
        )
);

TRACE_EVENT(api_stop_tx_ba_cb,
        TP_PROTO(struct ieee80211_sub_if_data *sdata, const u8 *ra, u16 tid),

        TP_ARGS(sdata, ra, tid),

        TP_STRUCT__entry(
                VIF_ENTRY
                __array(u8, ra, ETH_ALEN)
                __field(u16, tid)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                memcpy(__entry->ra, ra, ETH_ALEN);
                __entry->tid = tid;
        ),

        TP_printk(
                VIF_PR_FMT " ra:%pM tid:%d",
                VIF_PR_ARG, __entry->ra, __entry->tid
        )
);

DEFINE_EVENT(local_only_evt, api_restart_hw,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(api_beacon_loss,
        TP_PROTO(struct ieee80211_sub_if_data *sdata),

        TP_ARGS(sdata),

        TP_STRUCT__entry(
                VIF_ENTRY
        ),

        TP_fast_assign(
                VIF_ASSIGN;
        ),

        TP_printk(
                VIF_PR_FMT,
                VIF_PR_ARG
        )
);

TRACE_EVENT(api_connection_loss,
        TP_PROTO(struct ieee80211_sub_if_data *sdata),

        TP_ARGS(sdata),

        TP_STRUCT__entry(
                VIF_ENTRY
        ),

        TP_fast_assign(
                VIF_ASSIGN;
        ),

        TP_printk(
                VIF_PR_FMT,
                VIF_PR_ARG
        )
);

TRACE_EVENT(api_disconnect,
        TP_PROTO(struct ieee80211_sub_if_data *sdata, bool reconnect),

        TP_ARGS(sdata, reconnect),

        TP_STRUCT__entry(
                VIF_ENTRY
                __field(int, reconnect)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                __entry->reconnect = reconnect;
        ),

        TP_printk(
                VIF_PR_FMT " reconnect:%d",
                VIF_PR_ARG, __entry->reconnect
        )
);

TRACE_EVENT(api_cqm_rssi_notify,
        TP_PROTO(struct ieee80211_sub_if_data *sdata,
                 enum nl80211_cqm_rssi_threshold_event rssi_event,
                 s32 rssi_level),

        TP_ARGS(sdata, rssi_event, rssi_level),

        TP_STRUCT__entry(
                VIF_ENTRY
                __field(u32, rssi_event)
                __field(s32, rssi_level)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                __entry->rssi_event = rssi_event;
                __entry->rssi_level = rssi_level;
        ),

        TP_printk(
                VIF_PR_FMT " event:%d rssi:%d",
                VIF_PR_ARG, __entry->rssi_event, __entry->rssi_level
        )
);

DEFINE_EVENT(local_sdata_evt, api_cqm_beacon_loss_notify,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata),
        TP_ARGS(local, sdata)
);

TRACE_EVENT(api_scan_completed,
        TP_PROTO(struct ieee80211_local *local, bool aborted),

        TP_ARGS(local, aborted),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, aborted)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->aborted = aborted;
        ),

        TP_printk(
                LOCAL_PR_FMT " aborted:%d",
                LOCAL_PR_ARG, __entry->aborted
        )
);

TRACE_EVENT(api_sched_scan_results,
        TP_PROTO(struct ieee80211_local *local),

        TP_ARGS(local),

        TP_STRUCT__entry(
                LOCAL_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT, LOCAL_PR_ARG
        )
);

TRACE_EVENT(api_sched_scan_stopped,
        TP_PROTO(struct ieee80211_local *local),

        TP_ARGS(local),

        TP_STRUCT__entry(
                LOCAL_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT, LOCAL_PR_ARG
        )
);

TRACE_EVENT(api_sta_block_awake,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta, bool block),

        TP_ARGS(local, sta, block),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(bool, block)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->block = block;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT " block:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->block
        )
);

TRACE_EVENT(api_chswitch_done,
        TP_PROTO(struct ieee80211_sub_if_data *sdata, bool success,
                 unsigned int link_id),

        TP_ARGS(sdata, success, link_id),

        TP_STRUCT__entry(
                VIF_ENTRY
                __field(bool, success)
                __field(unsigned int, link_id)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                __entry->success = success;
                __entry->link_id = link_id;
        ),

        TP_printk(
                VIF_PR_FMT " success=%d link_id=%d",
                VIF_PR_ARG, __entry->success, __entry->link_id
        )
);

DEFINE_EVENT(local_only_evt, api_ready_on_channel,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

DEFINE_EVENT(local_only_evt, api_remain_on_channel_expired,
        TP_PROTO(struct ieee80211_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(api_gtk_rekey_notify,
        TP_PROTO(struct ieee80211_sub_if_data *sdata,
                 const u8 *bssid, const u8 *replay_ctr),

        TP_ARGS(sdata, bssid, replay_ctr),

        TP_STRUCT__entry(
                VIF_ENTRY
                __array(u8, bssid, ETH_ALEN)
                __array(u8, replay_ctr, NL80211_REPLAY_CTR_LEN)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                memcpy(__entry->bssid, bssid, ETH_ALEN);
                memcpy(__entry->replay_ctr, replay_ctr, NL80211_REPLAY_CTR_LEN);
        ),

        TP_printk(VIF_PR_FMT, VIF_PR_ARG)
);

TRACE_EVENT(api_enable_rssi_reports,
        TP_PROTO(struct ieee80211_sub_if_data *sdata,
                 int rssi_min_thold, int rssi_max_thold),

        TP_ARGS(sdata, rssi_min_thold, rssi_max_thold),

        TP_STRUCT__entry(
                VIF_ENTRY
                __field(int, rssi_min_thold)
                __field(int, rssi_max_thold)
        ),

        TP_fast_assign(
                VIF_ASSIGN;
                __entry->rssi_min_thold = rssi_min_thold;
                __entry->rssi_max_thold = rssi_max_thold;
        ),

        TP_printk(
                VIF_PR_FMT " rssi_min_thold =%d, rssi_max_thold = %d",
                VIF_PR_ARG, __entry->rssi_min_thold, __entry->rssi_max_thold
        )
);

TRACE_EVENT(api_eosp,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta),

        TP_ARGS(local, sta),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT,
                LOCAL_PR_ARG, STA_PR_ARG
        )
);

TRACE_EVENT(api_send_eosp_nullfunc,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 u8 tid),

        TP_ARGS(local, sta, tid),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(u8, tid)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->tid = tid;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT " tid:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->tid
        )
);

TRACE_EVENT(api_sta_set_buffered,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sta *sta,
                 u8 tid, bool buffered),

        TP_ARGS(local, sta, tid, buffered),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                STA_ENTRY
                __field(u8, tid)
                __field(bool, buffered)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                STA_ASSIGN;
                __entry->tid = tid;
                __entry->buffered = buffered;
        ),

        TP_printk(
                LOCAL_PR_FMT STA_PR_FMT " tid:%d buffered:%d",
                LOCAL_PR_ARG, STA_PR_ARG, __entry->tid, __entry->buffered
        )
);

TRACE_EVENT(api_radar_detected,
        TP_PROTO(struct ieee80211_local *local),

        TP_ARGS(local),

        TP_STRUCT__entry(
                LOCAL_ENTRY
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
        ),

        TP_printk(
                LOCAL_PR_FMT " radar detected",
                LOCAL_PR_ARG
        )
);

TRACE_EVENT(api_request_smps,
        TP_PROTO(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_link_data *link,
                 enum ieee80211_smps_mode smps_mode),

        TP_ARGS(local, sdata, link, smps_mode),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                VIF_ENTRY
                __field(int, link_id)
                __field(u32, smps_mode)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                VIF_ASSIGN;
                __entry->link_id = link->link_id,
                __entry->smps_mode = smps_mode;
        ),

        TP_printk(
                LOCAL_PR_FMT " " VIF_PR_FMT " link:%d, smps_mode:%d",
                LOCAL_PR_ARG, VIF_PR_ARG, __entry->link_id, __entry->smps_mode
        )
);

/*
 * Tracing for internal functions
 * (which may also be called in response to driver calls)
 */

TRACE_EVENT(wake_queue,
        TP_PROTO(struct ieee80211_local *local, u16 queue,
                 enum queue_stop_reason reason),

        TP_ARGS(local, queue, reason),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u16, queue)
                __field(u32, reason)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->queue = queue;
                __entry->reason = reason;
        ),

        TP_printk(
                LOCAL_PR_FMT " queue:%d, reason:%d",
                LOCAL_PR_ARG, __entry->queue, __entry->reason
        )
);

TRACE_EVENT(stop_queue,
        TP_PROTO(struct ieee80211_local *local, u16 queue,
                 enum queue_stop_reason reason),

        TP_ARGS(local, queue, reason),

        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u16, queue)
                __field(u32, reason)
        ),

        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->queue = queue;
                __entry->reason = reason;
        ),

        TP_printk(
                LOCAL_PR_FMT " queue:%d, reason:%d",
                LOCAL_PR_ARG, __entry->queue, __entry->reason
        )
);

TRACE_EVENT(drv_can_neg_ttlm,
            TP_PROTO(struct ieee80211_local *local,
                     struct ieee80211_sub_if_data *sdata,
                     struct ieee80211_neg_ttlm *neg_ttlm),

        TP_ARGS(local, sdata, neg_ttlm),

        TP_STRUCT__entry(LOCAL_ENTRY
                         VIF_ENTRY
                         __array(u16, downlink, sizeof(u16) * 8)
                         __array(u16, uplink, sizeof(u16) * 8)
        ),

        TP_fast_assign(LOCAL_ASSIGN;
                       VIF_ASSIGN;
                       memcpy(__entry->downlink, neg_ttlm->downlink,
                              sizeof(neg_ttlm->downlink));
                       memcpy(__entry->uplink, neg_ttlm->uplink,
                              sizeof(neg_ttlm->uplink));
        ),

        TP_printk(LOCAL_PR_FMT ", " VIF_PR_FMT, LOCAL_PR_ARG, VIF_PR_ARG)
);

TRACE_EVENT(drv_neg_ttlm_res,
            TP_PROTO(struct ieee80211_local *local,
                     struct ieee80211_sub_if_data *sdata,
                     enum ieee80211_neg_ttlm_res res,
                     struct ieee80211_neg_ttlm *neg_ttlm),

        TP_ARGS(local, sdata, res, neg_ttlm),

        TP_STRUCT__entry(LOCAL_ENTRY
                         VIF_ENTRY
                         __field(u32, res)
                         __array(u16, downlink, sizeof(u16) * 8)
                         __array(u16, uplink, sizeof(u16) * 8)
        ),

        TP_fast_assign(LOCAL_ASSIGN;
                       VIF_ASSIGN;
                       __entry->res = res;
                       memcpy(__entry->downlink, neg_ttlm->downlink,
                              sizeof(neg_ttlm->downlink));
                       memcpy(__entry->uplink, neg_ttlm->uplink,
                              sizeof(neg_ttlm->uplink));
        ),

        TP_printk(LOCAL_PR_FMT  VIF_PR_FMT " response: %d\n ",
                  LOCAL_PR_ARG, VIF_PR_ARG, __entry->res
        )
);

TRACE_EVENT(drv_prep_add_interface,
            TP_PROTO(struct ieee80211_local *local,
                     enum nl80211_iftype type),

        TP_ARGS(local, type),
        TP_STRUCT__entry(LOCAL_ENTRY
                         __field(u32, type)
        ),

        TP_fast_assign(LOCAL_ASSIGN;
                       __entry->type = type;
        ),

        TP_printk(LOCAL_PR_FMT  " type: %u\n ",
                  LOCAL_PR_ARG, __entry->type
        )
);

#endif /* !__MAC80211_DRIVER_TRACE || TRACE_HEADER_MULTI_READ */

#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE trace
#include <trace/define_trace.h>














 3159 

























































 3159 







































































 3164 






  103 






  593 




















































































  103 













  593 





































  103 






  593 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM net

#if !defined(_TRACE_NET_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_NET_H

#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/if_vlan.h>
#include <linux/ip.h>
#include <linux/tracepoint.h>

TRACE_EVENT(net_dev_start_xmit,

        TP_PROTO(const struct sk_buff *skb, const struct net_device *dev),

        TP_ARGS(skb, dev),

        TP_STRUCT__entry(
                __string(        name,                        dev->name        )
                __field(        u16,                        queue_mapping        )
                __field(        const void *,                skbaddr                )
                __field(        bool,                        vlan_tagged        )
                __field(        u16,                        vlan_proto        )
                __field(        u16,                        vlan_tci        )
                __field(        u16,                        protocol        )
                __field(        u8,                        ip_summed        )
                __field(        unsigned int,                len                )
                __field(        unsigned int,                data_len        )
                __field(        int,                        network_offset        )
                __field(        bool,                        transport_offset_valid)
                __field(        int,                        transport_offset)
                __field(        u8,                        tx_flags        )
                __field(        u16,                        gso_size        )
                __field(        u16,                        gso_segs        )
                __field(        u16,                        gso_type        )
        ),

        TP_fast_assign(
                __assign_str(name);
                __entry->queue_mapping = skb->queue_mapping;
                __entry->skbaddr = skb;
                __entry->vlan_tagged = skb_vlan_tag_present(skb);
                __entry->vlan_proto = ntohs(skb->vlan_proto);
                __entry->vlan_tci = skb_vlan_tag_get(skb);
                __entry->protocol = ntohs(skb->protocol);
                __entry->ip_summed = skb->ip_summed;
                __entry->len = skb->len;
                __entry->data_len = skb->data_len;
                __entry->network_offset = skb_network_offset(skb);
                __entry->transport_offset_valid =
                        skb_transport_header_was_set(skb);
                __entry->transport_offset = skb_transport_header_was_set(skb) ?
                        skb_transport_offset(skb) : 0;
                __entry->tx_flags = skb_shinfo(skb)->tx_flags;
                __entry->gso_size = skb_shinfo(skb)->gso_size;
                __entry->gso_segs = skb_shinfo(skb)->gso_segs;
                __entry->gso_type = skb_shinfo(skb)->gso_type;
        ),

        TP_printk("dev=%s queue_mapping=%u skbaddr=%p vlan_tagged=%d vlan_proto=0x%04x vlan_tci=0x%04x protocol=0x%04x ip_summed=%d len=%u data_len=%u network_offset=%d transport_offset_valid=%d transport_offset=%d tx_flags=%d gso_size=%d gso_segs=%d gso_type=%#x",
                  __get_str(name), __entry->queue_mapping, __entry->skbaddr,
                  __entry->vlan_tagged, __entry->vlan_proto, __entry->vlan_tci,
                  __entry->protocol, __entry->ip_summed, __entry->len,
                  __entry->data_len,
                  __entry->network_offset, __entry->transport_offset_valid,
                  __entry->transport_offset, __entry->tx_flags,
                  __entry->gso_size, __entry->gso_segs, __entry->gso_type)
);

TRACE_EVENT(net_dev_xmit,

        TP_PROTO(struct sk_buff *skb,
                 int rc,
                 struct net_device *dev,
                 unsigned int skb_len),

        TP_ARGS(skb, rc, dev, skb_len),

        TP_STRUCT__entry(
                __field(        void *,                skbaddr                )
                __field(        unsigned int,        len                )
                __field(        int,                rc                )
                __string(        name,                dev->name        )
        ),

        TP_fast_assign(
                __entry->skbaddr = skb;
                __entry->len = skb_len;
                __entry->rc = rc;
                __assign_str(name);
        ),

        TP_printk("dev=%s skbaddr=%p len=%u rc=%d",
                __get_str(name), __entry->skbaddr, __entry->len, __entry->rc)
);

TRACE_EVENT(net_dev_xmit_timeout,

        TP_PROTO(struct net_device *dev,
                 int queue_index),

        TP_ARGS(dev, queue_index),

        TP_STRUCT__entry(
                __string(        name,                dev->name        )
                __string(        driver,                netdev_drivername(dev))
                __field(        int,                queue_index        )
        ),

        TP_fast_assign(
                __assign_str(name);
                __assign_str(driver);
                __entry->queue_index = queue_index;
        ),

        TP_printk("dev=%s driver=%s queue=%d",
                __get_str(name), __get_str(driver), __entry->queue_index)
);

DECLARE_EVENT_CLASS(net_dev_template,

        TP_PROTO(struct sk_buff *skb),

        TP_ARGS(skb),

        TP_STRUCT__entry(
                __field(        void *,                skbaddr                )
                __field(        unsigned int,        len                )
                __string(        name,                skb->dev->name        )
        ),

        TP_fast_assign(
                __entry->skbaddr = skb;
                __entry->len = skb->len;
                __assign_str(name);
        ),

        TP_printk("dev=%s skbaddr=%p len=%u",
                __get_str(name), __entry->skbaddr, __entry->len)
)

DEFINE_EVENT(net_dev_template, net_dev_queue,

        TP_PROTO(struct sk_buff *skb),

        TP_ARGS(skb)
);

DEFINE_EVENT(net_dev_template, netif_receive_skb,

        TP_PROTO(struct sk_buff *skb),

        TP_ARGS(skb)
);

DEFINE_EVENT(net_dev_template, netif_rx,

        TP_PROTO(struct sk_buff *skb),

        TP_ARGS(skb)
);

DECLARE_EVENT_CLASS(net_dev_rx_verbose_template,

        TP_PROTO(const struct sk_buff *skb),

        TP_ARGS(skb),

        TP_STRUCT__entry(
                __string(        name,                        skb->dev->name        )
                __field(        unsigned int,                napi_id                )
                __field(        u16,                        queue_mapping        )
                __field(        const void *,                skbaddr                )
                __field(        bool,                        vlan_tagged        )
                __field(        u16,                        vlan_proto        )
                __field(        u16,                        vlan_tci        )
                __field(        u16,                        protocol        )
                __field(        u8,                        ip_summed        )
                __field(        u32,                        hash                )
                __field(        bool,                        l4_hash                )
                __field(        unsigned int,                len                )
                __field(        unsigned int,                data_len        )
                __field(        unsigned int,                truesize        )
                __field(        bool,                        mac_header_valid)
                __field(        int,                        mac_header        )
                __field(        unsigned char,                nr_frags        )
                __field(        u16,                        gso_size        )
                __field(        u16,                        gso_type        )
        ),

        TP_fast_assign(
                __assign_str(name);
#ifdef CONFIG_NET_RX_BUSY_POLL
                __entry->napi_id = skb->napi_id;
#else
                __entry->napi_id = 0;
#endif
                __entry->queue_mapping = skb->queue_mapping;
                __entry->skbaddr = skb;
                __entry->vlan_tagged = skb_vlan_tag_present(skb);
                __entry->vlan_proto = ntohs(skb->vlan_proto);
                __entry->vlan_tci = skb_vlan_tag_get(skb);
                __entry->protocol = ntohs(skb->protocol);
                __entry->ip_summed = skb->ip_summed;
                __entry->hash = skb->hash;
                __entry->l4_hash = skb->l4_hash;
                __entry->len = skb->len;
                __entry->data_len = skb->data_len;
                __entry->truesize = skb->truesize;
                __entry->mac_header_valid = skb_mac_header_was_set(skb);
                __entry->mac_header = skb_mac_header(skb) - skb->data;
                __entry->nr_frags = skb_shinfo(skb)->nr_frags;
                __entry->gso_size = skb_shinfo(skb)->gso_size;
                __entry->gso_type = skb_shinfo(skb)->gso_type;
        ),

        TP_printk("dev=%s napi_id=%#x queue_mapping=%u skbaddr=%p vlan_tagged=%d vlan_proto=0x%04x vlan_tci=0x%04x protocol=0x%04x ip_summed=%d hash=0x%08x l4_hash=%d len=%u data_len=%u truesize=%u mac_header_valid=%d mac_header=%d nr_frags=%d gso_size=%d gso_type=%#x",
                  __get_str(name), __entry->napi_id, __entry->queue_mapping,
                  __entry->skbaddr, __entry->vlan_tagged, __entry->vlan_proto,
                  __entry->vlan_tci, __entry->protocol, __entry->ip_summed,
                  __entry->hash, __entry->l4_hash, __entry->len,
                  __entry->data_len, __entry->truesize,
                  __entry->mac_header_valid, __entry->mac_header,
                  __entry->nr_frags, __entry->gso_size, __entry->gso_type)
);

DEFINE_EVENT(net_dev_rx_verbose_template, napi_gro_frags_entry,

        TP_PROTO(const struct sk_buff *skb),

        TP_ARGS(skb)
);

DEFINE_EVENT(net_dev_rx_verbose_template, napi_gro_receive_entry,

        TP_PROTO(const struct sk_buff *skb),

        TP_ARGS(skb)
);

DEFINE_EVENT(net_dev_rx_verbose_template, netif_receive_skb_entry,

        TP_PROTO(const struct sk_buff *skb),

        TP_ARGS(skb)
);

DEFINE_EVENT(net_dev_rx_verbose_template, netif_receive_skb_list_entry,

        TP_PROTO(const struct sk_buff *skb),

        TP_ARGS(skb)
);

DEFINE_EVENT(net_dev_rx_verbose_template, netif_rx_entry,

        TP_PROTO(const struct sk_buff *skb),

        TP_ARGS(skb)
);

DECLARE_EVENT_CLASS(net_dev_rx_exit_template,

        TP_PROTO(int ret),

        TP_ARGS(ret),

        TP_STRUCT__entry(
                __field(int,        ret)
        ),

        TP_fast_assign(
                __entry->ret = ret;
        ),

        TP_printk("ret=%d", __entry->ret)
);

DEFINE_EVENT(net_dev_rx_exit_template, napi_gro_frags_exit,

        TP_PROTO(int ret),

        TP_ARGS(ret)
);

DEFINE_EVENT(net_dev_rx_exit_template, napi_gro_receive_exit,

        TP_PROTO(int ret),

        TP_ARGS(ret)
);

DEFINE_EVENT(net_dev_rx_exit_template, netif_receive_skb_exit,

        TP_PROTO(int ret),

        TP_ARGS(ret)
);

DEFINE_EVENT(net_dev_rx_exit_template, netif_rx_exit,

        TP_PROTO(int ret),

        TP_ARGS(ret)
);

DEFINE_EVENT(net_dev_rx_exit_template, netif_receive_skb_list_exit,

        TP_PROTO(int ret),

        TP_ARGS(ret)
);

#endif /* _TRACE_NET_H */

/* This part must be outside protection */
#include <trace/define_trace.h>



















































































  249 

  246 
  248 

  249 






  205 
  247 


  247 
  248 






    4 
    1 
















































  672 




  673 

  674 
  675 







  673 


  668 








   27 
   23 
















   24 
















  197 







  196 








  196 


























   22 














    9 

    9 
    3 













    6 













    6 



    9 


    6 
    6 




























  121 



































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Ethernet-type device handling.
 *
 * Version:        @(#)eth.c        1.0.7        05/25/93
 *
 * Authors:        Ross Biro
 *                Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *                Mark Evans, <evansmp@uhura.aston.ac.uk>
 *                Florian  La Roche, <rzsfl@rz.uni-sb.de>
 *                Alan Cox, <gw4pts@gw4pts.ampr.org>
 *
 * Fixes:
 *                Mr Linux        : Arp problems
 *                Alan Cox        : Generic queue tidyup (very tiny here)
 *                Alan Cox        : eth_header ntohs should be htons
 *                Alan Cox        : eth_rebuild_header missing an htons and
 *                                  minor other things.
 *                Tegge                : Arp bug fixes.
 *                Florian                : Removed many unnecessary functions, code cleanup
 *                                  and changes for new arp and skbuff.
 *                Alan Cox        : Redid header building to reflect new format.
 *                Alan Cox        : ARP only when compiled with CONFIG_INET
 *                Greg Page        : 802.2 and SNAP stuff.
 *                Alan Cox        : MAC layer pointers/new format.
 *                Paul Gortmaker        : eth_copy_and_sum shouldn't csum padding.
 *                Alan Cox        : Protect against forwarding explosions with
 *                                  older network drivers and IFF_ALLMULTI.
 *        Christer Weinigel        : Better rebuild header message.
 *             Andrew Morton    : 26Feb01: kill ether_setup() - use netdev_boot_setup().
 */
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/ip.h>
#include <linux/netdevice.h>
#include <linux/nvmem-consumer.h>
#include <linux/etherdevice.h>
#include <linux/skbuff.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/if_ether.h>
#include <linux/of_net.h>
#include <linux/pci.h>
#include <linux/property.h>
#include <net/dst.h>
#include <net/arp.h>
#include <net/sock.h>
#include <net/ipv6.h>
#include <net/ip.h>
#include <net/dsa.h>
#include <net/flow_dissector.h>
#include <net/gro.h>
#include <linux/uaccess.h>
#include <net/pkt_sched.h>

/**
 * eth_header - create the Ethernet header
 * @skb:        buffer to alter
 * @dev:        source device
 * @type:        Ethernet type field
 * @daddr: destination address (NULL leave destination address)
 * @saddr: source address (NULL use device source address)
 * @len:   packet length (<= skb->len)
 *
 *
 * Set the protocol type. For a packet of type ETH_P_802_3/2 we put the length
 * in here instead.
 */
int eth_header(struct sk_buff *skb, struct net_device *dev,
               unsigned short type,
               const void *daddr, const void *saddr, unsigned int len)
{
        struct ethhdr *eth = skb_push(skb, ETH_HLEN);

        if (type != ETH_P_802_3 && type != ETH_P_802_2)
                eth->h_proto = htons(type);
        else
                eth->h_proto = htons(len);

        /*
         *      Set the source hardware address.
         */

        if (!saddr)
                saddr = dev->dev_addr;
        memcpy(eth->h_source, saddr, ETH_ALEN);

        if (daddr) {
                memcpy(eth->h_dest, daddr, ETH_ALEN);
                return ETH_HLEN;
        }

        /*
         *      Anyway, the loopback-device should never use this function...
         */

        if (dev->flags & (IFF_LOOPBACK | IFF_NOARP)) {
                eth_zero_addr(eth->h_dest);
                return ETH_HLEN;
        }

        return -ETH_HLEN;
}
EXPORT_SYMBOL(eth_header);

/**
 * eth_get_headlen - determine the length of header for an ethernet frame
 * @dev: pointer to network device
 * @data: pointer to start of frame
 * @len: total length of frame
 *
 * Make a best effort attempt to pull the length for all of the headers for
 * a given frame in a linear buffer.
 */
u32 eth_get_headlen(const struct net_device *dev, const void *data, u32 len)
{
        const unsigned int flags = FLOW_DISSECTOR_F_PARSE_1ST_FRAG;
        const struct ethhdr *eth = (const struct ethhdr *)data;
        struct flow_keys_basic keys;

        /* this should never happen, but better safe than sorry */
        if (unlikely(len < sizeof(*eth)))
                return len;

        /* parse any remaining L2/L3 headers, check for L4 */
        if (!skb_flow_dissect_flow_keys_basic(dev_net(dev), NULL, &keys, data,
                                              eth->h_proto, sizeof(*eth),
                                              len, flags))
                return max_t(u32, keys.control.thoff, sizeof(*eth));

        /* parse for any L4 headers */
        return min_t(u32, __skb_get_poff(NULL, data, &keys, len), len);
}
EXPORT_SYMBOL(eth_get_headlen);

/**
 * eth_type_trans - determine the packet's protocol ID.
 * @skb: received socket data
 * @dev: receiving network device
 *
 * The rule here is that we
 * assume 802.3 if the type field is short enough to be a length.
 * This is normal practice and works for any 'now in use' protocol.
 */
__be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev)
{
        unsigned short _service_access_point;
        const unsigned short *sap;
        const struct ethhdr *eth;

        skb->dev = dev;
        skb_reset_mac_header(skb);

        eth = eth_skb_pull_mac(skb);
        eth_skb_pkt_type(skb, dev);

        /*
         * Some variants of DSA tagging don't have an ethertype field
         * at all, so we check here whether one of those tagging
         * variants has been configured on the receiving interface,
         * and if so, set skb->protocol without looking at the packet.
         */
        if (unlikely(netdev_uses_dsa(dev)))
                return htons(ETH_P_XDSA);

        if (likely(eth_proto_is_802_3(eth->h_proto)))
                return eth->h_proto;

        /*
         *      This is a magic hack to spot IPX packets. Older Novell breaks
         *      the protocol design and runs IPX over 802.3 without an 802.2 LLC
         *      layer. We look for FFFF which isn't a used 802.2 SSAP/DSAP. This
         *      won't work for fault tolerant netware but does for the rest.
         */
        sap = skb_header_pointer(skb, 0, sizeof(*sap), &_service_access_point);
        if (sap && *sap == 0xFFFF)
                return htons(ETH_P_802_3);

        /*
         *      Real 802.2 LLC
         */
        return htons(ETH_P_802_2);
}
EXPORT_SYMBOL(eth_type_trans);

/**
 * eth_header_parse - extract hardware address from packet
 * @skb: packet to extract header from
 * @haddr: destination buffer
 */
int eth_header_parse(const struct sk_buff *skb, unsigned char *haddr)
{
        const struct ethhdr *eth = eth_hdr(skb);
        memcpy(haddr, eth->h_source, ETH_ALEN);
        return ETH_ALEN;
}
EXPORT_SYMBOL(eth_header_parse);

/**
 * eth_header_cache - fill cache entry from neighbour
 * @neigh: source neighbour
 * @hh: destination cache entry
 * @type: Ethernet type field
 *
 * Create an Ethernet header template from the neighbour.
 */
int eth_header_cache(const struct neighbour *neigh, struct hh_cache *hh, __be16 type)
{
        struct ethhdr *eth;
        const struct net_device *dev = neigh->dev;

        eth = (struct ethhdr *)
            (((u8 *) hh->hh_data) + (HH_DATA_OFF(sizeof(*eth))));

        if (type == htons(ETH_P_802_3))
                return -1;

        eth->h_proto = type;
        memcpy(eth->h_source, dev->dev_addr, ETH_ALEN);
        memcpy(eth->h_dest, neigh->ha, ETH_ALEN);

        /* Pairs with READ_ONCE() in neigh_resolve_output(),
         * neigh_hh_output() and neigh_update_hhs().
         */
        smp_store_release(&hh->hh_len, ETH_HLEN);

        return 0;
}
EXPORT_SYMBOL(eth_header_cache);

/**
 * eth_header_cache_update - update cache entry
 * @hh: destination cache entry
 * @dev: network device
 * @haddr: new hardware address
 *
 * Called by Address Resolution module to notify changes in address.
 */
void eth_header_cache_update(struct hh_cache *hh,
                             const struct net_device *dev,
                             const unsigned char *haddr)
{
        memcpy(((u8 *) hh->hh_data) + HH_DATA_OFF(sizeof(struct ethhdr)),
               haddr, ETH_ALEN);
}
EXPORT_SYMBOL(eth_header_cache_update);

/**
 * eth_header_parse_protocol - extract protocol from L2 header
 * @skb: packet to extract protocol from
 */
__be16 eth_header_parse_protocol(const struct sk_buff *skb)
{
        const struct ethhdr *eth = eth_hdr(skb);

        return eth->h_proto;
}
EXPORT_SYMBOL(eth_header_parse_protocol);

/**
 * eth_prepare_mac_addr_change - prepare for mac change
 * @dev: network device
 * @p: socket address
 */
int eth_prepare_mac_addr_change(struct net_device *dev, void *p)
{
        struct sockaddr *addr = p;

        if (!(dev->priv_flags & IFF_LIVE_ADDR_CHANGE) && netif_running(dev))
                return -EBUSY;
        if (!is_valid_ether_addr(addr->sa_data))
                return -EADDRNOTAVAIL;
        return 0;
}
EXPORT_SYMBOL(eth_prepare_mac_addr_change);

/**
 * eth_commit_mac_addr_change - commit mac change
 * @dev: network device
 * @p: socket address
 */
void eth_commit_mac_addr_change(struct net_device *dev, void *p)
{
        struct sockaddr *addr = p;

        eth_hw_addr_set(dev, addr->sa_data);
}
EXPORT_SYMBOL(eth_commit_mac_addr_change);

/**
 * eth_mac_addr - set new Ethernet hardware address
 * @dev: network device
 * @p: socket address
 *
 * Change hardware address of device.
 *
 * This doesn't change hardware matching, so needs to be overridden
 * for most real devices.
 */
int eth_mac_addr(struct net_device *dev, void *p)
{
        int ret;

        ret = eth_prepare_mac_addr_change(dev, p);
        if (ret < 0)
                return ret;
        eth_commit_mac_addr_change(dev, p);
        return 0;
}
EXPORT_SYMBOL(eth_mac_addr);

int eth_validate_addr(struct net_device *dev)
{
        if (!is_valid_ether_addr(dev->dev_addr))
                return -EADDRNOTAVAIL;

        return 0;
}
EXPORT_SYMBOL(eth_validate_addr);

const struct header_ops eth_header_ops ____cacheline_aligned = {
        .create                = eth_header,
        .parse                = eth_header_parse,
        .cache                = eth_header_cache,
        .cache_update        = eth_header_cache_update,
        .parse_protocol        = eth_header_parse_protocol,
};

/**
 * ether_setup - setup Ethernet network device
 * @dev: network device
 *
 * Fill in the fields of the device structure with Ethernet-generic values.
 */
void ether_setup(struct net_device *dev)
{
        dev->header_ops                = &eth_header_ops;
        dev->type                = ARPHRD_ETHER;
        dev->hard_header_len         = ETH_HLEN;
        dev->min_header_len        = ETH_HLEN;
        dev->mtu                = ETH_DATA_LEN;
        dev->min_mtu                = ETH_MIN_MTU;
        dev->max_mtu                = ETH_DATA_LEN;
        dev->addr_len                = ETH_ALEN;
        dev->tx_queue_len        = DEFAULT_TX_QUEUE_LEN;
        dev->flags                = IFF_BROADCAST|IFF_MULTICAST;
        dev->priv_flags                |= IFF_TX_SKB_SHARING;

        eth_broadcast_addr(dev->broadcast);

}
EXPORT_SYMBOL(ether_setup);

/**
 * alloc_etherdev_mqs - Allocates and sets up an Ethernet device
 * @sizeof_priv: Size of additional driver-private structure to be allocated
 *        for this Ethernet device
 * @txqs: The number of TX queues this device has.
 * @rxqs: The number of RX queues this device has.
 *
 * Fill in the fields of the device structure with Ethernet-generic
 * values. Basically does everything except registering the device.
 *
 * Constructs a new net device, complete with a private data area of
 * size (sizeof_priv).  A 32-byte (not bit) alignment is enforced for
 * this private data area.
 */

struct net_device *alloc_etherdev_mqs(int sizeof_priv, unsigned int txqs,
                                      unsigned int rxqs)
{
        return alloc_netdev_mqs(sizeof_priv, "eth%d", NET_NAME_ENUM,
                                ether_setup, txqs, rxqs);
}
EXPORT_SYMBOL(alloc_etherdev_mqs);

ssize_t sysfs_format_mac(char *buf, const unsigned char *addr, int len)
{
        return sysfs_emit(buf, "%*phC\n", len, addr);
}
EXPORT_SYMBOL(sysfs_format_mac);

struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb)
{
        const struct packet_offload *ptype;
        unsigned int hlen, off_eth;
        struct sk_buff *pp = NULL;
        struct ethhdr *eh, *eh2;
        struct sk_buff *p;
        __be16 type;
        int flush = 1;

        off_eth = skb_gro_offset(skb);
        hlen = off_eth + sizeof(*eh);
        eh = skb_gro_header(skb, hlen, off_eth);
        if (unlikely(!eh))
                goto out;

        flush = 0;

        list_for_each_entry(p, head, list) {
                if (!NAPI_GRO_CB(p)->same_flow)
                        continue;

                eh2 = (struct ethhdr *)(p->data + off_eth);
                if (compare_ether_header(eh, eh2)) {
                        NAPI_GRO_CB(p)->same_flow = 0;
                        continue;
                }
        }

        type = eh->h_proto;

        ptype = gro_find_receive_by_type(type);
        if (ptype == NULL) {
                flush = 1;
                goto out;
        }

        skb_gro_pull(skb, sizeof(*eh));
        skb_gro_postpull_rcsum(skb, eh, sizeof(*eh));

        pp = indirect_call_gro_receive_inet(ptype->callbacks.gro_receive,
                                            ipv6_gro_receive, inet_gro_receive,
                                            head, skb);

out:
        skb_gro_flush_final(skb, pp, flush);

        return pp;
}
EXPORT_SYMBOL(eth_gro_receive);

int eth_gro_complete(struct sk_buff *skb, int nhoff)
{
        struct ethhdr *eh = (struct ethhdr *)(skb->data + nhoff);
        __be16 type = eh->h_proto;
        struct packet_offload *ptype;
        int err = -ENOSYS;

        if (skb->encapsulation)
                skb_set_inner_mac_header(skb, nhoff);

        ptype = gro_find_complete_by_type(type);
        if (ptype != NULL)
                err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
                                         ipv6_gro_complete, inet_gro_complete,
                                         skb, nhoff + sizeof(*eh));

        return err;
}
EXPORT_SYMBOL(eth_gro_complete);

static struct packet_offload eth_packet_offload __read_mostly = {
        .type = cpu_to_be16(ETH_P_TEB),
        .priority = 10,
        .callbacks = {
                .gro_receive = eth_gro_receive,
                .gro_complete = eth_gro_complete,
        },
};

static int __init eth_offload_init(void)
{
        dev_add_offload(&eth_packet_offload);

        return 0;
}

fs_initcall(eth_offload_init);

unsigned char * __weak arch_get_platform_mac_address(void)
{
        return NULL;
}

int eth_platform_get_mac_address(struct device *dev, u8 *mac_addr)
{
        unsigned char *addr;
        int ret;

        ret = of_get_mac_address(dev->of_node, mac_addr);
        if (!ret)
                return 0;

        addr = arch_get_platform_mac_address();
        if (!addr)
                return -ENODEV;

        ether_addr_copy(mac_addr, addr);

        return 0;
}
EXPORT_SYMBOL(eth_platform_get_mac_address);

/**
 * platform_get_ethdev_address - Set netdev's MAC address from a given device
 * @dev:        Pointer to the device
 * @netdev:        Pointer to netdev to write the address to
 *
 * Wrapper around eth_platform_get_mac_address() which writes the address
 * directly to netdev->dev_addr.
 */
int platform_get_ethdev_address(struct device *dev, struct net_device *netdev)
{
        u8 addr[ETH_ALEN] __aligned(2);
        int ret;

        ret = eth_platform_get_mac_address(dev, addr);
        if (!ret)
                eth_hw_addr_set(netdev, addr);
        return ret;
}
EXPORT_SYMBOL(platform_get_ethdev_address);

/**
 * nvmem_get_mac_address - Obtain the MAC address from an nvmem cell named
 * 'mac-address' associated with given device.
 *
 * @dev:        Device with which the mac-address cell is associated.
 * @addrbuf:        Buffer to which the MAC address will be copied on success.
 *
 * Returns 0 on success or a negative error number on failure.
 */
int nvmem_get_mac_address(struct device *dev, void *addrbuf)
{
        struct nvmem_cell *cell;
        const void *mac;
        size_t len;

        cell = nvmem_cell_get(dev, "mac-address");
        if (IS_ERR(cell))
                return PTR_ERR(cell);

        mac = nvmem_cell_read(cell, &len);
        nvmem_cell_put(cell);

        if (IS_ERR(mac))
                return PTR_ERR(mac);

        if (len != ETH_ALEN || !is_valid_ether_addr(mac)) {
                kfree(mac);
                return -EINVAL;
        }

        ether_addr_copy(addrbuf, mac);
        kfree(mac);

        return 0;
}

static int fwnode_get_mac_addr(struct fwnode_handle *fwnode,
                               const char *name, char *addr)
{
        int ret;

        ret = fwnode_property_read_u8_array(fwnode, name, addr, ETH_ALEN);
        if (ret)
                return ret;

        if (!is_valid_ether_addr(addr))
                return -EINVAL;
        return 0;
}

/**
 * fwnode_get_mac_address - Get the MAC from the firmware node
 * @fwnode:        Pointer to the firmware node
 * @addr:        Address of buffer to store the MAC in
 *
 * Search the firmware node for the best MAC address to use.  'mac-address' is
 * checked first, because that is supposed to contain to "most recent" MAC
 * address. If that isn't set, then 'local-mac-address' is checked next,
 * because that is the default address.  If that isn't set, then the obsolete
 * 'address' is checked, just in case we're using an old device tree.
 *
 * Note that the 'address' property is supposed to contain a virtual address of
 * the register set, but some DTS files have redefined that property to be the
 * MAC address.
 *
 * All-zero MAC addresses are rejected, because those could be properties that
 * exist in the firmware tables, but were not updated by the firmware.  For
 * example, the DTS could define 'mac-address' and 'local-mac-address', with
 * zero MAC addresses.  Some older U-Boots only initialized 'local-mac-address'.
 * In this case, the real MAC is in 'local-mac-address', and 'mac-address'
 * exists but is all zeros.
 */
int fwnode_get_mac_address(struct fwnode_handle *fwnode, char *addr)
{
        if (!fwnode_get_mac_addr(fwnode, "mac-address", addr) ||
            !fwnode_get_mac_addr(fwnode, "local-mac-address", addr) ||
            !fwnode_get_mac_addr(fwnode, "address", addr))
                return 0;

        return -ENOENT;
}
EXPORT_SYMBOL(fwnode_get_mac_address);

/**
 * device_get_mac_address - Get the MAC for a given device
 * @dev:        Pointer to the device
 * @addr:        Address of buffer to store the MAC in
 */
int device_get_mac_address(struct device *dev, char *addr)
{
        return fwnode_get_mac_address(dev_fwnode(dev), addr);
}
EXPORT_SYMBOL(device_get_mac_address);

/**
 * device_get_ethdev_address - Set netdev's MAC address from a given device
 * @dev:        Pointer to the device
 * @netdev:        Pointer to netdev to write the address to
 *
 * Wrapper around device_get_mac_address() which writes the address
 * directly to netdev->dev_addr.
 */
int device_get_ethdev_address(struct device *dev, struct net_device *netdev)
{
        u8 addr[ETH_ALEN];
        int ret;

        ret = device_get_mac_address(dev, addr);
        if (!ret)
                eth_hw_addr_set(netdev, addr);
        return ret;
}
EXPORT_SYMBOL(device_get_ethdev_address);
























    4 





   35 

   35 
   35 










   35 








   35 






   35 












   35 
   35 




   35 








   35 
   35 



   34 






   35 









   35 


    4 

   35 

   35 













   34 


    4 

   35 


   34 














   36 










   36 



   36 































   39 
   39 
   39 
   10 

   39 
   37 

   39 











































































































   10 


















































































































































    4 











    4 


    4 






    4 




    4 











    4 
    4 
    4 

    4 





































   35 


   35 


   35 






   34 






   35 








   35 








































































    4 








    4 





















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  net/dccp/output.c
 *
 *  An implementation of the DCCP protocol
 *  Arnaldo Carvalho de Melo <acme@conectiva.com.br>
 */

#include <linux/dccp.h>
#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/sched/signal.h>

#include <net/inet_sock.h>
#include <net/sock.h>

#include "ackvec.h"
#include "ccid.h"
#include "dccp.h"

static inline void dccp_event_ack_sent(struct sock *sk)
{
        inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
}

/* enqueue @skb on sk_send_head for retransmission, return clone to send now */
static struct sk_buff *dccp_skb_entail(struct sock *sk, struct sk_buff *skb)
{
        skb_set_owner_w(skb, sk);
        WARN_ON(sk->sk_send_head);
        sk->sk_send_head = skb;
        return skb_clone(sk->sk_send_head, gfp_any());
}

/*
 * All SKB's seen here are completely headerless. It is our
 * job to build the DCCP header, and pass the packet down to
 * IP so it can do the same plus pass the packet off to the
 * device.
 */
static int dccp_transmit_skb(struct sock *sk, struct sk_buff *skb)
{
        if (likely(skb != NULL)) {
                struct inet_sock *inet = inet_sk(sk);
                const struct inet_connection_sock *icsk = inet_csk(sk);
                struct dccp_sock *dp = dccp_sk(sk);
                struct dccp_skb_cb *dcb = DCCP_SKB_CB(skb);
                struct dccp_hdr *dh;
                /* XXX For now we're using only 48 bits sequence numbers */
                const u32 dccp_header_size = sizeof(*dh) +
                                             sizeof(struct dccp_hdr_ext) +
                                          dccp_packet_hdr_len(dcb->dccpd_type);
                int err, set_ack = 1;
                u64 ackno = dp->dccps_gsr;
                /*
                 * Increment GSS here already in case the option code needs it.
                 * Update GSS for real only if option processing below succeeds.
                 */
                dcb->dccpd_seq = ADD48(dp->dccps_gss, 1);

                switch (dcb->dccpd_type) {
                case DCCP_PKT_DATA:
                        set_ack = 0;
                        fallthrough;
                case DCCP_PKT_DATAACK:
                case DCCP_PKT_RESET:
                        break;

                case DCCP_PKT_REQUEST:
                        set_ack = 0;
                        /* Use ISS on the first (non-retransmitted) Request. */
                        if (icsk->icsk_retransmits == 0)
                                dcb->dccpd_seq = dp->dccps_iss;
                        fallthrough;

                case DCCP_PKT_SYNC:
                case DCCP_PKT_SYNCACK:
                        ackno = dcb->dccpd_ack_seq;
                        fallthrough;
                default:
                        /*
                         * Set owner/destructor: some skbs are allocated via
                         * alloc_skb (e.g. when retransmission may happen).
                         * Only Data, DataAck, and Reset packets should come
                         * through here with skb->sk set.
                         */
                        WARN_ON(skb->sk);
                        skb_set_owner_w(skb, sk);
                        break;
                }

                if (dccp_insert_options(sk, skb)) {
                        kfree_skb(skb);
                        return -EPROTO;
                }


                /* Build DCCP header and checksum it. */
                dh = dccp_zeroed_hdr(skb, dccp_header_size);
                dh->dccph_type        = dcb->dccpd_type;
                dh->dccph_sport        = inet->inet_sport;
                dh->dccph_dport        = inet->inet_dport;
                dh->dccph_doff        = (dccp_header_size + dcb->dccpd_opt_len) / 4;
                dh->dccph_ccval        = dcb->dccpd_ccval;
                dh->dccph_cscov = dp->dccps_pcslen;
                /* XXX For now we're using only 48 bits sequence numbers */
                dh->dccph_x        = 1;

                dccp_update_gss(sk, dcb->dccpd_seq);
                dccp_hdr_set_seq(dh, dp->dccps_gss);
                if (set_ack)
                        dccp_hdr_set_ack(dccp_hdr_ack_bits(skb), ackno);

                switch (dcb->dccpd_type) {
                case DCCP_PKT_REQUEST:
                        dccp_hdr_request(skb)->dccph_req_service =
                                                        dp->dccps_service;
                        /*
                         * Limit Ack window to ISS <= P.ackno <= GSS, so that
                         * only Responses to Requests we sent are considered.
                         */
                        dp->dccps_awl = dp->dccps_iss;
                        break;
                case DCCP_PKT_RESET:
                        dccp_hdr_reset(skb)->dccph_reset_code =
                                                        dcb->dccpd_reset_code;
                        break;
                }

                icsk->icsk_af_ops->send_check(sk, skb);

                if (set_ack)
                        dccp_event_ack_sent(sk);

                DCCP_INC_STATS(DCCP_MIB_OUTSEGS);

                err = icsk->icsk_af_ops->queue_xmit(sk, skb, &inet->cork.fl);
                return net_xmit_eval(err);
        }
        return -ENOBUFS;
}

/**
 * dccp_determine_ccmps  -  Find out about CCID-specific packet-size limits
 * @dp: socket to find packet size limits of
 *
 * We only consider the HC-sender CCID for setting the CCMPS (RFC 4340, 14.),
 * since the RX CCID is restricted to feedback packets (Acks), which are small
 * in comparison with the data traffic. A value of 0 means "no current CCMPS".
 */
static u32 dccp_determine_ccmps(const struct dccp_sock *dp)
{
        const struct ccid *tx_ccid = dp->dccps_hc_tx_ccid;

        if (tx_ccid == NULL || tx_ccid->ccid_ops == NULL)
                return 0;
        return tx_ccid->ccid_ops->ccid_ccmps;
}

unsigned int dccp_sync_mss(struct sock *sk, u32 pmtu)
{
        struct inet_connection_sock *icsk = inet_csk(sk);
        struct dccp_sock *dp = dccp_sk(sk);
        u32 ccmps = dccp_determine_ccmps(dp);
        u32 cur_mps = ccmps ? min(pmtu, ccmps) : pmtu;

        /* Account for header lengths and IPv4/v6 option overhead */
        cur_mps -= (icsk->icsk_af_ops->net_header_len + icsk->icsk_ext_hdr_len +
                    sizeof(struct dccp_hdr) + sizeof(struct dccp_hdr_ext));

        /*
         * Leave enough headroom for common DCCP header options.
         * This only considers options which may appear on DCCP-Data packets, as
         * per table 3 in RFC 4340, 5.8. When running out of space for other
         * options (eg. Ack Vector which can take up to 255 bytes), it is better
         * to schedule a separate Ack. Thus we leave headroom for the following:
         *  - 1 byte for Slow Receiver (11.6)
         *  - 6 bytes for Timestamp (13.1)
         *  - 10 bytes for Timestamp Echo (13.3)
         *  - 8 bytes for NDP count (7.7, when activated)
         *  - 6 bytes for Data Checksum (9.3)
         *  - %DCCPAV_MIN_OPTLEN bytes for Ack Vector size (11.4, when enabled)
         */
        cur_mps -= roundup(1 + 6 + 10 + dp->dccps_send_ndp_count * 8 + 6 +
                           (dp->dccps_hc_rx_ackvec ? DCCPAV_MIN_OPTLEN : 0), 4);

        /* And store cached results */
        icsk->icsk_pmtu_cookie = pmtu;
        WRITE_ONCE(dp->dccps_mss_cache, cur_mps);

        return cur_mps;
}

EXPORT_SYMBOL_GPL(dccp_sync_mss);

void dccp_write_space(struct sock *sk)
{
        struct socket_wq *wq;

        rcu_read_lock();
        wq = rcu_dereference(sk->sk_wq);
        if (skwq_has_sleeper(wq))
                wake_up_interruptible(&wq->wait);
        /* Should agree with poll, otherwise some programs break */
        if (sock_writeable(sk))
                sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);

        rcu_read_unlock();
}

/**
 * dccp_wait_for_ccid  -  Await CCID send permission
 * @sk:    socket to wait for
 * @delay: timeout in jiffies
 *
 * This is used by CCIDs which need to delay the send time in process context.
 */
static int dccp_wait_for_ccid(struct sock *sk, unsigned long delay)
{
        DEFINE_WAIT(wait);
        long remaining;

        prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
        sk->sk_write_pending++;
        release_sock(sk);

        remaining = schedule_timeout(delay);

        lock_sock(sk);
        sk->sk_write_pending--;
        finish_wait(sk_sleep(sk), &wait);

        if (signal_pending(current) || sk->sk_err)
                return -1;
        return remaining;
}

/**
 * dccp_xmit_packet  -  Send data packet under control of CCID
 * @sk: socket to send data packet on
 *
 * Transmits next-queued payload and informs CCID to account for the packet.
 */
static void dccp_xmit_packet(struct sock *sk)
{
        int err, len;
        struct dccp_sock *dp = dccp_sk(sk);
        struct sk_buff *skb = dccp_qpolicy_pop(sk);

        if (unlikely(skb == NULL))
                return;
        len = skb->len;

        if (sk->sk_state == DCCP_PARTOPEN) {
                const u32 cur_mps = dp->dccps_mss_cache - DCCP_FEATNEG_OVERHEAD;
                /*
                 * See 8.1.5 - Handshake Completion.
                 *
                 * For robustness we resend Confirm options until the client has
                 * entered OPEN. During the initial feature negotiation, the MPS
                 * is smaller than usual, reduced by the Change/Confirm options.
                 */
                if (!list_empty(&dp->dccps_featneg) && len > cur_mps) {
                        DCCP_WARN("Payload too large (%d) for featneg.\n", len);
                        dccp_send_ack(sk);
                        dccp_feat_list_purge(&dp->dccps_featneg);
                }

                inet_csk_schedule_ack(sk);
                inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
                                              inet_csk(sk)->icsk_rto,
                                              DCCP_RTO_MAX);
                DCCP_SKB_CB(skb)->dccpd_type = DCCP_PKT_DATAACK;
        } else if (dccp_ack_pending(sk)) {
                DCCP_SKB_CB(skb)->dccpd_type = DCCP_PKT_DATAACK;
        } else {
                DCCP_SKB_CB(skb)->dccpd_type = DCCP_PKT_DATA;
        }

        err = dccp_transmit_skb(sk, skb);
        if (err)
                dccp_pr_debug("transmit_skb() returned err=%d\n", err);
        /*
         * Register this one as sent even if an error occurred. To the remote
         * end a local packet drop is indistinguishable from network loss, i.e.
         * any local drop will eventually be reported via receiver feedback.
         */
        ccid_hc_tx_packet_sent(dp->dccps_hc_tx_ccid, sk, len);

        /*
         * If the CCID needs to transfer additional header options out-of-band
         * (e.g. Ack Vectors or feature-negotiation options), it activates this
         * flag to schedule a Sync. The Sync will automatically incorporate all
         * currently pending header options, thus clearing the backlog.
         */
        if (dp->dccps_sync_scheduled)
                dccp_send_sync(sk, dp->dccps_gsr, DCCP_PKT_SYNC);
}

/**
 * dccp_flush_write_queue  -  Drain queue at end of connection
 * @sk: socket to be drained
 * @time_budget: time allowed to drain the queue
 *
 * Since dccp_sendmsg queues packets without waiting for them to be sent, it may
 * happen that the TX queue is not empty at the end of a connection. We give the
 * HC-sender CCID a grace period of up to @time_budget jiffies. If this function
 * returns with a non-empty write queue, it will be purged later.
 */
void dccp_flush_write_queue(struct sock *sk, long *time_budget)
{
        struct dccp_sock *dp = dccp_sk(sk);
        struct sk_buff *skb;
        long delay, rc;

        while (*time_budget > 0 && (skb = skb_peek(&sk->sk_write_queue))) {
                rc = ccid_hc_tx_send_packet(dp->dccps_hc_tx_ccid, sk, skb);

                switch (ccid_packet_dequeue_eval(rc)) {
                case CCID_PACKET_WILL_DEQUEUE_LATER:
                        /*
                         * If the CCID determines when to send, the next sending
                         * time is unknown or the CCID may not even send again
                         * (e.g. remote host crashes or lost Ack packets).
                         */
                        DCCP_WARN("CCID did not manage to send all packets\n");
                        return;
                case CCID_PACKET_DELAY:
                        delay = msecs_to_jiffies(rc);
                        if (delay > *time_budget)
                                return;
                        rc = dccp_wait_for_ccid(sk, delay);
                        if (rc < 0)
                                return;
                        *time_budget -= (delay - rc);
                        /* check again if we can send now */
                        break;
                case CCID_PACKET_SEND_AT_ONCE:
                        dccp_xmit_packet(sk);
                        break;
                case CCID_PACKET_ERR:
                        skb_dequeue(&sk->sk_write_queue);
                        kfree_skb(skb);
                        dccp_pr_debug("packet discarded due to err=%ld\n", rc);
                }
        }
}

void dccp_write_xmit(struct sock *sk)
{
        struct dccp_sock *dp = dccp_sk(sk);
        struct sk_buff *skb;

        while ((skb = dccp_qpolicy_top(sk))) {
                int rc = ccid_hc_tx_send_packet(dp->dccps_hc_tx_ccid, sk, skb);

                switch (ccid_packet_dequeue_eval(rc)) {
                case CCID_PACKET_WILL_DEQUEUE_LATER:
                        return;
                case CCID_PACKET_DELAY:
                        sk_reset_timer(sk, &dp->dccps_xmit_timer,
                                       jiffies + msecs_to_jiffies(rc));
                        return;
                case CCID_PACKET_SEND_AT_ONCE:
                        dccp_xmit_packet(sk);
                        break;
                case CCID_PACKET_ERR:
                        dccp_qpolicy_drop(sk, skb);
                        dccp_pr_debug("packet discarded due to err=%d\n", rc);
                }
        }
}

/**
 * dccp_retransmit_skb  -  Retransmit Request, Close, or CloseReq packets
 * @sk: socket to perform retransmit on
 *
 * There are only four retransmittable packet types in DCCP:
 * - Request  in client-REQUEST  state (sec. 8.1.1),
 * - CloseReq in server-CLOSEREQ state (sec. 8.3),
 * - Close    in   node-CLOSING  state (sec. 8.3),
 * - Acks in client-PARTOPEN state (sec. 8.1.5, handled by dccp_delack_timer()).
 * This function expects sk->sk_send_head to contain the original skb.
 */
int dccp_retransmit_skb(struct sock *sk)
{
        WARN_ON(sk->sk_send_head == NULL);

        if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk) != 0)
                return -EHOSTUNREACH; /* Routing failure or similar. */

        /* this count is used to distinguish original and retransmitted skb */
        inet_csk(sk)->icsk_retransmits++;

        return dccp_transmit_skb(sk, skb_clone(sk->sk_send_head, GFP_ATOMIC));
}

struct sk_buff *dccp_make_response(const struct sock *sk, struct dst_entry *dst,
                                   struct request_sock *req)
{
        struct dccp_hdr *dh;
        struct dccp_request_sock *dreq;
        const u32 dccp_header_size = sizeof(struct dccp_hdr) +
                                     sizeof(struct dccp_hdr_ext) +
                                     sizeof(struct dccp_hdr_response);
        struct sk_buff *skb;

        /* sk is marked const to clearly express we dont hold socket lock.
         * sock_wmalloc() will atomically change sk->sk_wmem_alloc,
         * it is safe to promote sk to non const.
         */
        skb = sock_wmalloc((struct sock *)sk, MAX_DCCP_HEADER, 1,
                           GFP_ATOMIC);
        if (!skb)
                return NULL;

        skb_reserve(skb, MAX_DCCP_HEADER);

        skb_dst_set(skb, dst_clone(dst));

        dreq = dccp_rsk(req);
        if (inet_rsk(req)->acked)        /* increase GSS upon retransmission */
                dccp_inc_seqno(&dreq->dreq_gss);
        DCCP_SKB_CB(skb)->dccpd_type = DCCP_PKT_RESPONSE;
        DCCP_SKB_CB(skb)->dccpd_seq  = dreq->dreq_gss;

        /* Resolve feature dependencies resulting from choice of CCID */
        if (dccp_feat_server_ccid_dependencies(dreq))
                goto response_failed;

        if (dccp_insert_options_rsk(dreq, skb))
                goto response_failed;

        /* Build and checksum header */
        dh = dccp_zeroed_hdr(skb, dccp_header_size);

        dh->dccph_sport        = htons(inet_rsk(req)->ir_num);
        dh->dccph_dport        = inet_rsk(req)->ir_rmt_port;
        dh->dccph_doff        = (dccp_header_size +
                           DCCP_SKB_CB(skb)->dccpd_opt_len) / 4;
        dh->dccph_type        = DCCP_PKT_RESPONSE;
        dh->dccph_x        = 1;
        dccp_hdr_set_seq(dh, dreq->dreq_gss);
        dccp_hdr_set_ack(dccp_hdr_ack_bits(skb), dreq->dreq_gsr);
        dccp_hdr_response(skb)->dccph_resp_service = dreq->dreq_service;

        dccp_csum_outgoing(skb);

        /* We use `acked' to remember that a Response was already sent. */
        inet_rsk(req)->acked = 1;
        DCCP_INC_STATS(DCCP_MIB_OUTSEGS);
        return skb;
response_failed:
        kfree_skb(skb);
        return NULL;
}

EXPORT_SYMBOL_GPL(dccp_make_response);

/* answer offending packet in @rcv_skb with Reset from control socket @ctl */
struct sk_buff *dccp_ctl_make_reset(struct sock *sk, struct sk_buff *rcv_skb)
{
        struct dccp_hdr *rxdh = dccp_hdr(rcv_skb), *dh;
        struct dccp_skb_cb *dcb = DCCP_SKB_CB(rcv_skb);
        const u32 dccp_hdr_reset_len = sizeof(struct dccp_hdr) +
                                       sizeof(struct dccp_hdr_ext) +
                                       sizeof(struct dccp_hdr_reset);
        struct dccp_hdr_reset *dhr;
        struct sk_buff *skb;

        skb = alloc_skb(sk->sk_prot->max_header, GFP_ATOMIC);
        if (skb == NULL)
                return NULL;

        skb_reserve(skb, sk->sk_prot->max_header);

        /* Swap the send and the receive. */
        dh = dccp_zeroed_hdr(skb, dccp_hdr_reset_len);
        dh->dccph_type        = DCCP_PKT_RESET;
        dh->dccph_sport        = rxdh->dccph_dport;
        dh->dccph_dport        = rxdh->dccph_sport;
        dh->dccph_doff        = dccp_hdr_reset_len / 4;
        dh->dccph_x        = 1;

        dhr = dccp_hdr_reset(skb);
        dhr->dccph_reset_code = dcb->dccpd_reset_code;

        switch (dcb->dccpd_reset_code) {
        case DCCP_RESET_CODE_PACKET_ERROR:
                dhr->dccph_reset_data[0] = rxdh->dccph_type;
                break;
        case DCCP_RESET_CODE_OPTION_ERROR:
        case DCCP_RESET_CODE_MANDATORY_ERROR:
                memcpy(dhr->dccph_reset_data, dcb->dccpd_reset_data, 3);
                break;
        }
        /*
         * From RFC 4340, 8.3.1:
         *   If P.ackno exists, set R.seqno := P.ackno + 1.
         *   Else set R.seqno := 0.
         */
        if (dcb->dccpd_ack_seq != DCCP_PKT_WITHOUT_ACK_SEQ)
                dccp_hdr_set_seq(dh, ADD48(dcb->dccpd_ack_seq, 1));
        dccp_hdr_set_ack(dccp_hdr_ack_bits(skb), dcb->dccpd_seq);

        dccp_csum_outgoing(skb);
        return skb;
}

EXPORT_SYMBOL_GPL(dccp_ctl_make_reset);

/* send Reset on established socket, to close or abort the connection */
int dccp_send_reset(struct sock *sk, enum dccp_reset_codes code)
{
        struct sk_buff *skb;
        /*
         * FIXME: what if rebuild_header fails?
         * Should we be doing a rebuild_header here?
         */
        int err = inet_csk(sk)->icsk_af_ops->rebuild_header(sk);

        if (err != 0)
                return err;

        skb = sock_wmalloc(sk, sk->sk_prot->max_header, 1, GFP_ATOMIC);
        if (skb == NULL)
                return -ENOBUFS;

        /* Reserve space for headers and prepare control bits. */
        skb_reserve(skb, sk->sk_prot->max_header);
        DCCP_SKB_CB(skb)->dccpd_type           = DCCP_PKT_RESET;
        DCCP_SKB_CB(skb)->dccpd_reset_code = code;

        return dccp_transmit_skb(sk, skb);
}

/*
 * Do all connect socket setups that can be done AF independent.
 */
int dccp_connect(struct sock *sk)
{
        struct sk_buff *skb;
        struct dccp_sock *dp = dccp_sk(sk);
        struct dst_entry *dst = __sk_dst_get(sk);
        struct inet_connection_sock *icsk = inet_csk(sk);

        sk->sk_err = 0;
        sock_reset_flag(sk, SOCK_DONE);

        dccp_sync_mss(sk, dst_mtu(dst));

        /* do not connect if feature negotiation setup fails */
        if (dccp_feat_finalise_settings(dccp_sk(sk)))
                return -EPROTO;

        /* Initialise GAR as per 8.5; AWL/AWH are set in dccp_transmit_skb() */
        dp->dccps_gar = dp->dccps_iss;

        skb = alloc_skb(sk->sk_prot->max_header, sk->sk_allocation);
        if (unlikely(skb == NULL))
                return -ENOBUFS;

        /* Reserve space for headers. */
        skb_reserve(skb, sk->sk_prot->max_header);

        DCCP_SKB_CB(skb)->dccpd_type = DCCP_PKT_REQUEST;

        dccp_transmit_skb(sk, dccp_skb_entail(sk, skb));
        DCCP_INC_STATS(DCCP_MIB_ACTIVEOPENS);

        /* Timer for repeating the REQUEST until an answer. */
        icsk->icsk_retransmits = 0;
        inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
                                  icsk->icsk_rto, DCCP_RTO_MAX);
        return 0;
}

EXPORT_SYMBOL_GPL(dccp_connect);

void dccp_send_ack(struct sock *sk)
{
        /* If we have been reset, we may not send again. */
        if (sk->sk_state != DCCP_CLOSED) {
                struct sk_buff *skb = alloc_skb(sk->sk_prot->max_header,
                                                GFP_ATOMIC);

                if (skb == NULL) {
                        inet_csk_schedule_ack(sk);
                        inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN;
                        inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
                                                  TCP_DELACK_MAX,
                                                  DCCP_RTO_MAX);
                        return;
                }

                /* Reserve space for headers */
                skb_reserve(skb, sk->sk_prot->max_header);
                DCCP_SKB_CB(skb)->dccpd_type = DCCP_PKT_ACK;
                dccp_transmit_skb(sk, skb);
        }
}

EXPORT_SYMBOL_GPL(dccp_send_ack);

#if 0
/* FIXME: Is this still necessary (11.3) - currently nowhere used by DCCP. */
void dccp_send_delayed_ack(struct sock *sk)
{
        struct inet_connection_sock *icsk = inet_csk(sk);
        /*
         * FIXME: tune this timer. elapsed time fixes the skew, so no problem
         * with using 2s, and active senders also piggyback the ACK into a
         * DATAACK packet, so this is really for quiescent senders.
         */
        unsigned long timeout = jiffies + 2 * HZ;

        /* Use new timeout only if there wasn't a older one earlier. */
        if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
                /* If delack timer was blocked or is about to expire,
                 * send ACK now.
                 *
                 * FIXME: check the "about to expire" part
                 */
                if (icsk->icsk_ack.blocked) {
                        dccp_send_ack(sk);
                        return;
                }

                if (!time_before(timeout, icsk->icsk_ack.timeout))
                        timeout = icsk->icsk_ack.timeout;
        }
        icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
        icsk->icsk_ack.timeout = timeout;
        sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
}
#endif

void dccp_send_sync(struct sock *sk, const u64 ackno,
                    const enum dccp_pkt_type pkt_type)
{
        /*
         * We are not putting this on the write queue, so
         * dccp_transmit_skb() will set the ownership to this
         * sock.
         */
        struct sk_buff *skb = alloc_skb(sk->sk_prot->max_header, GFP_ATOMIC);

        if (skb == NULL) {
                /* FIXME: how to make sure the sync is sent? */
                DCCP_CRIT("could not send %s", dccp_packet_name(pkt_type));
                return;
        }

        /* Reserve space for headers and prepare control bits. */
        skb_reserve(skb, sk->sk_prot->max_header);
        DCCP_SKB_CB(skb)->dccpd_type = pkt_type;
        DCCP_SKB_CB(skb)->dccpd_ack_seq = ackno;

        /*
         * Clear the flag in case the Sync was scheduled for out-of-band data,
         * such as carrying a long Ack Vector.
         */
        dccp_sk(sk)->dccps_sync_scheduled = 0;

        dccp_transmit_skb(sk, skb);
}

EXPORT_SYMBOL_GPL(dccp_send_sync);

/*
 * Send a DCCP_PKT_CLOSE/CLOSEREQ. The caller locks the socket for us. This
 * cannot be allowed to fail queueing a DCCP_PKT_CLOSE/CLOSEREQ frame under
 * any circumstances.
 */
void dccp_send_close(struct sock *sk, const int active)
{
        struct dccp_sock *dp = dccp_sk(sk);
        struct sk_buff *skb;
        const gfp_t prio = active ? GFP_KERNEL : GFP_ATOMIC;

        skb = alloc_skb(sk->sk_prot->max_header, prio);
        if (skb == NULL)
                return;

        /* Reserve space for headers and prepare control bits. */
        skb_reserve(skb, sk->sk_prot->max_header);
        if (dp->dccps_role == DCCP_ROLE_SERVER && !dp->dccps_server_timewait)
                DCCP_SKB_CB(skb)->dccpd_type = DCCP_PKT_CLOSEREQ;
        else
                DCCP_SKB_CB(skb)->dccpd_type = DCCP_PKT_CLOSE;

        if (active) {
                skb = dccp_skb_entail(sk, skb);
                /*
                 * Retransmission timer for active-close: RFC 4340, 8.3 requires
                 * to retransmit the Close/CloseReq until the CLOSING/CLOSEREQ
                 * state can be left. The initial timeout is 2 RTTs.
                 * Since RTT measurement is done by the CCIDs, there is no easy
                 * way to get an RTT sample. The fallback RTT from RFC 4340, 3.4
                 * is too low (200ms); we use a high value to avoid unnecessary
                 * retransmissions when the link RTT is > 0.2 seconds.
                 * FIXME: Let main module sample RTTs and use that instead.
                 */
                inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
                                          DCCP_TIMEOUT_INIT, DCCP_RTO_MAX);
        }
        dccp_transmit_skb(sk, skb);
}




























































































































































   70 
   76 





   18 
    5 
    7 

    7 


    5 


   13 












    5 


    5 




    7 



    7 

    6 




    5 
    5 


    2 

    5 





















































    4 




    4 


















    4 


    4 



    4 











    4 

    4 


    4 























































   11 

















    4 


    4 






































    4 






























































































   11 


   11 

   11 










   11 





   11 










   11 


    5 



   11 
   11 


    5 
    5 
    5 


    5 



























    4 















    4 




   17 





   17 



















   11 



   11 
   11 

   11 



   11 



    4 
   11 






   11 





   11 






   11 




   11 





    8 




    8 








































































































    4 


    4 



    4 





    4 

    4 







    4 
    4 

    4 






    4 









    4 
    4 
    4 
    4 










































    4 
    4 
    4 






























































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
// SPDX-License-Identifier: GPL-2.0
/*
 * Central processing for nfsd.
 *
 * Authors:        Olaf Kirch (okir@monad.swb.de)
 *
 * Copyright (C) 1995, 1996, 1997 Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/sched/signal.h>
#include <linux/freezer.h>
#include <linux/module.h>
#include <linux/fs_struct.h>
#include <linux/swap.h>
#include <linux/siphash.h>

#include <linux/sunrpc/stats.h>
#include <linux/sunrpc/svcsock.h>
#include <linux/sunrpc/svc_xprt.h>
#include <linux/lockd/bind.h>
#include <linux/nfsacl.h>
#include <linux/nfslocalio.h>
#include <linux/seq_file.h>
#include <linux/inetdevice.h>
#include <net/addrconf.h>
#include <net/ipv6.h>
#include <net/net_namespace.h>
#include "nfsd.h"
#include "cache.h"
#include "vfs.h"
#include "netns.h"
#include "filecache.h"

#include "trace.h"

#define NFSDDBG_FACILITY        NFSDDBG_SVC

atomic_t                        nfsd_th_cnt = ATOMIC_INIT(0);
static int                        nfsd(void *vrqstp);
#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
static int                        nfsd_acl_rpcbind_set(struct net *,
                                                     const struct svc_program *,
                                                     u32, int,
                                                     unsigned short,
                                                     unsigned short);
static __be32                        nfsd_acl_init_request(struct svc_rqst *,
                                                const struct svc_program *,
                                                struct svc_process_info *);
#endif
static int                        nfsd_rpcbind_set(struct net *,
                                                 const struct svc_program *,
                                                 u32, int,
                                                 unsigned short,
                                                 unsigned short);
static __be32                        nfsd_init_request(struct svc_rqst *,
                                                const struct svc_program *,
                                                struct svc_process_info *);

/*
 * nfsd_mutex protects nn->nfsd_serv -- both the pointer itself and some members
 * of the svc_serv struct such as ->sv_temp_socks and ->sv_permsocks.
 *
 * Finally, the nfsd_mutex also protects some of the global variables that are
 * accessed when nfsd starts and that are settable via the write_* routines in
 * nfsctl.c. In particular:
 *
 *        user_recovery_dirname
 *        user_lease_time
 *        nfsd_versions
 */
DEFINE_MUTEX(nfsd_mutex);

/*
 * nfsd_drc_lock protects nfsd_drc_max_pages and nfsd_drc_pages_used.
 * nfsd_drc_max_pages limits the total amount of memory available for
 * version 4.1 DRC caches.
 * nfsd_drc_pages_used tracks the current version 4.1 DRC memory usage.
 */
DEFINE_SPINLOCK(nfsd_drc_lock);
unsigned long        nfsd_drc_max_mem;
unsigned long        nfsd_drc_mem_used;

#if IS_ENABLED(CONFIG_NFS_LOCALIO)
static const struct svc_version *localio_versions[] = {
        [1] = &localio_version1,
};

#define NFSD_LOCALIO_NRVERS                ARRAY_SIZE(localio_versions)

#endif /* CONFIG_NFS_LOCALIO */

#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
static const struct svc_version *nfsd_acl_version[] = {
# if defined(CONFIG_NFSD_V2_ACL)
        [2] = &nfsd_acl_version2,
# endif
# if defined(CONFIG_NFSD_V3_ACL)
        [3] = &nfsd_acl_version3,
# endif
};

#define NFSD_ACL_MINVERS        2
#define NFSD_ACL_NRVERS                ARRAY_SIZE(nfsd_acl_version)

#endif /* defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) */

static const struct svc_version *nfsd_version[NFSD_MAXVERS+1] = {
#if defined(CONFIG_NFSD_V2)
        [2] = &nfsd_version2,
#endif
        [3] = &nfsd_version3,
#if defined(CONFIG_NFSD_V4)
        [4] = &nfsd_version4,
#endif
};

struct svc_program                nfsd_programs[] = {
        {
        .pg_prog                = NFS_PROGRAM,                /* program number */
        .pg_nvers                = NFSD_MAXVERS+1,        /* nr of entries in nfsd_version */
        .pg_vers                = nfsd_version,                /* version table */
        .pg_name                = "nfsd",                /* program name */
        .pg_class                = "nfsd",                /* authentication class */
        .pg_authenticate        = svc_set_client,        /* export authentication */
        .pg_init_request        = nfsd_init_request,
        .pg_rpcbind_set                = nfsd_rpcbind_set,
        },
#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
        {
        .pg_prog                = NFS_ACL_PROGRAM,
        .pg_nvers                = NFSD_ACL_NRVERS,
        .pg_vers                = nfsd_acl_version,
        .pg_name                = "nfsacl",
        .pg_class                = "nfsd",
        .pg_authenticate        = svc_set_client,
        .pg_init_request        = nfsd_acl_init_request,
        .pg_rpcbind_set                = nfsd_acl_rpcbind_set,
        },
#endif /* defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) */
#if IS_ENABLED(CONFIG_NFS_LOCALIO)
        {
        .pg_prog                = NFS_LOCALIO_PROGRAM,
        .pg_nvers                = NFSD_LOCALIO_NRVERS,
        .pg_vers                = localio_versions,
        .pg_name                = "nfslocalio",
        .pg_class                = "nfsd",
        .pg_authenticate        = svc_set_client,
        .pg_init_request        = svc_generic_init_request,
        .pg_rpcbind_set                = svc_generic_rpcbind_set,
        }
#endif /* CONFIG_NFS_LOCALIO */
};

bool nfsd_support_version(int vers)
{
        if (vers >= NFSD_MINVERS && vers <= NFSD_MAXVERS)
                return nfsd_version[vers] != NULL;
        return false;
}

int nfsd_vers(struct nfsd_net *nn, int vers, enum vers_op change)
{
        if (vers < NFSD_MINVERS || vers > NFSD_MAXVERS)
                return 0;
        switch(change) {
        case NFSD_SET:
                nn->nfsd_versions[vers] = nfsd_support_version(vers);
                break;
        case NFSD_CLEAR:
                nn->nfsd_versions[vers] = false;
                break;
        case NFSD_TEST:
                return nn->nfsd_versions[vers];
        case NFSD_AVAIL:
                return nfsd_support_version(vers);
        }
        return 0;
}

static void
nfsd_adjust_nfsd_versions4(struct nfsd_net *nn)
{
        unsigned i;

        for (i = 0; i <= NFSD_SUPPORTED_MINOR_VERSION; i++) {
                if (nn->nfsd4_minorversions[i])
                        return;
        }
        nfsd_vers(nn, 4, NFSD_CLEAR);
}

int nfsd_minorversion(struct nfsd_net *nn, u32 minorversion, enum vers_op change)
{
        if (minorversion > NFSD_SUPPORTED_MINOR_VERSION &&
            change != NFSD_AVAIL)
                return -1;

        switch(change) {
        case NFSD_SET:
                nfsd_vers(nn, 4, NFSD_SET);
                nn->nfsd4_minorversions[minorversion] =
                        nfsd_vers(nn, 4, NFSD_TEST);
                break;
        case NFSD_CLEAR:
                nn->nfsd4_minorversions[minorversion] = false;
                nfsd_adjust_nfsd_versions4(nn);
                break;
        case NFSD_TEST:
                return nn->nfsd4_minorversions[minorversion];
        case NFSD_AVAIL:
                return minorversion <= NFSD_SUPPORTED_MINOR_VERSION &&
                        nfsd_vers(nn, 4, NFSD_AVAIL);
        }
        return 0;
}

bool nfsd_serv_try_get(struct net *net) __must_hold(rcu)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        return (nn && percpu_ref_tryget_live(&nn->nfsd_serv_ref));
}

void nfsd_serv_put(struct net *net) __must_hold(rcu)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        percpu_ref_put(&nn->nfsd_serv_ref);
}

static void nfsd_serv_done(struct percpu_ref *ref)
{
        struct nfsd_net *nn = container_of(ref, struct nfsd_net, nfsd_serv_ref);

        complete(&nn->nfsd_serv_confirm_done);
}

static void nfsd_serv_free(struct percpu_ref *ref)
{
        struct nfsd_net *nn = container_of(ref, struct nfsd_net, nfsd_serv_ref);

        complete(&nn->nfsd_serv_free_done);
}

/*
 * Maximum number of nfsd processes
 */
#define        NFSD_MAXSERVS                8192

int nfsd_nrthreads(struct net *net)
{
        int rv = 0;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        mutex_lock(&nfsd_mutex);
        if (nn->nfsd_serv)
                rv = nn->nfsd_serv->sv_nrthreads;
        mutex_unlock(&nfsd_mutex);
        return rv;
}

static int nfsd_init_socks(struct net *net, const struct cred *cred)
{
        int error;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        if (!list_empty(&nn->nfsd_serv->sv_permsocks))
                return 0;

        error = svc_xprt_create(nn->nfsd_serv, "udp", net, PF_INET, NFS_PORT,
                                SVC_SOCK_DEFAULTS, cred);
        if (error < 0)
                return error;

        error = svc_xprt_create(nn->nfsd_serv, "tcp", net, PF_INET, NFS_PORT,
                                SVC_SOCK_DEFAULTS, cred);
        if (error < 0)
                return error;

        return 0;
}

static int nfsd_users = 0;

static int nfsd_startup_generic(void)
{
        int ret;

        if (nfsd_users++)
                return 0;

        ret = nfsd_file_cache_init();
        if (ret)
                goto dec_users;

        ret = nfs4_state_start();
        if (ret)
                goto out_file_cache;
        return 0;

out_file_cache:
        nfsd_file_cache_shutdown();
dec_users:
        nfsd_users--;
        return ret;
}

static void nfsd_shutdown_generic(void)
{
        if (--nfsd_users)
                return;

        nfs4_state_shutdown();
        nfsd_file_cache_shutdown();
}

static bool nfsd_needs_lockd(struct nfsd_net *nn)
{
        return nfsd_vers(nn, 2, NFSD_TEST) || nfsd_vers(nn, 3, NFSD_TEST);
}

/**
 * nfsd_copy_write_verifier - Atomically copy a write verifier
 * @verf: buffer in which to receive the verifier cookie
 * @nn: NFS net namespace
 *
 * This function provides a wait-free mechanism for copying the
 * namespace's write verifier without tearing it.
 */
void nfsd_copy_write_verifier(__be32 verf[2], struct nfsd_net *nn)
{
        unsigned int seq;

        do {
                seq = read_seqbegin(&nn->writeverf_lock);
                memcpy(verf, nn->writeverf, sizeof(nn->writeverf));
        } while (read_seqretry(&nn->writeverf_lock, seq));
}

static void nfsd_reset_write_verifier_locked(struct nfsd_net *nn)
{
        struct timespec64 now;
        u64 verf;

        /*
         * Because the time value is hashed, y2038 time_t overflow
         * is irrelevant in this usage.
         */
        ktime_get_raw_ts64(&now);
        verf = siphash_2u64(now.tv_sec, now.tv_nsec, &nn->siphash_key);
        memcpy(nn->writeverf, &verf, sizeof(nn->writeverf));
}

/**
 * nfsd_reset_write_verifier - Generate a new write verifier
 * @nn: NFS net namespace
 *
 * This function updates the ->writeverf field of @nn. This field
 * contains an opaque cookie that, according to Section 18.32.3 of
 * RFC 8881, "the client can use to determine whether a server has
 * changed instance state (e.g., server restart) between a call to
 * WRITE and a subsequent call to either WRITE or COMMIT.  This
 * cookie MUST be unchanged during a single instance of the NFSv4.1
 * server and MUST be unique between instances of the NFSv4.1
 * server."
 */
void nfsd_reset_write_verifier(struct nfsd_net *nn)
{
        write_seqlock(&nn->writeverf_lock);
        nfsd_reset_write_verifier_locked(nn);
        write_sequnlock(&nn->writeverf_lock);
}

/*
 * Crank up a set of per-namespace resources for a new NFSD instance,
 * including lockd, a duplicate reply cache, an open file cache
 * instance, and a cache of NFSv4 state objects.
 */
static int nfsd_startup_net(struct net *net, const struct cred *cred)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        int ret;

        if (nn->nfsd_net_up)
                return 0;

        ret = nfsd_startup_generic();
        if (ret)
                return ret;
        ret = nfsd_init_socks(net, cred);
        if (ret)
                goto out_socks;

        if (nfsd_needs_lockd(nn) && !nn->lockd_up) {
                ret = lockd_up(net, cred);
                if (ret)
                        goto out_socks;
                nn->lockd_up = true;
        }

        ret = nfsd_file_cache_start_net(net);
        if (ret)
                goto out_lockd;

        ret = nfsd_reply_cache_init(nn);
        if (ret)
                goto out_filecache;

        ret = nfs4_state_start_net(net);
        if (ret)
                goto out_reply_cache;

#ifdef CONFIG_NFSD_V4_2_INTER_SSC
        nfsd4_ssc_init_umount_work(nn);
#endif
        nn->nfsd_net_up = true;
        return 0;

out_reply_cache:
        nfsd_reply_cache_shutdown(nn);
out_filecache:
        nfsd_file_cache_shutdown_net(net);
out_lockd:
        if (nn->lockd_up) {
                lockd_down(net);
                nn->lockd_up = false;
        }
out_socks:
        nfsd_shutdown_generic();
        return ret;
}

static void nfsd_shutdown_net(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        if (!nn->nfsd_net_up)
                return;
        nfsd_export_flush(net);
        nfs4_state_shutdown_net(net);
        nfsd_reply_cache_shutdown(nn);
        nfsd_file_cache_shutdown_net(net);
        if (nn->lockd_up) {
                lockd_down(net);
                nn->lockd_up = false;
        }
        percpu_ref_exit(&nn->nfsd_serv_ref);
        nn->nfsd_net_up = false;
        nfsd_shutdown_generic();
}

static DEFINE_SPINLOCK(nfsd_notifier_lock);
static int nfsd_inetaddr_event(struct notifier_block *this, unsigned long event,
        void *ptr)
{
        struct in_ifaddr *ifa = (struct in_ifaddr *)ptr;
        struct net_device *dev = ifa->ifa_dev->dev;
        struct net *net = dev_net(dev);
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct sockaddr_in sin;

        if (event != NETDEV_DOWN || !nn->nfsd_serv)
                goto out;

        spin_lock(&nfsd_notifier_lock);
        if (nn->nfsd_serv) {
                dprintk("nfsd_inetaddr_event: removed %pI4\n", &ifa->ifa_local);
                sin.sin_family = AF_INET;
                sin.sin_addr.s_addr = ifa->ifa_local;
                svc_age_temp_xprts_now(nn->nfsd_serv, (struct sockaddr *)&sin);
        }
        spin_unlock(&nfsd_notifier_lock);

out:
        return NOTIFY_DONE;
}

static struct notifier_block nfsd_inetaddr_notifier = {
        .notifier_call = nfsd_inetaddr_event,
};

#if IS_ENABLED(CONFIG_IPV6)
static int nfsd_inet6addr_event(struct notifier_block *this,
        unsigned long event, void *ptr)
{
        struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)ptr;
        struct net_device *dev = ifa->idev->dev;
        struct net *net = dev_net(dev);
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct sockaddr_in6 sin6;

        if (event != NETDEV_DOWN || !nn->nfsd_serv)
                goto out;

        spin_lock(&nfsd_notifier_lock);
        if (nn->nfsd_serv) {
                dprintk("nfsd_inet6addr_event: removed %pI6\n", &ifa->addr);
                sin6.sin6_family = AF_INET6;
                sin6.sin6_addr = ifa->addr;
                if (ipv6_addr_type(&sin6.sin6_addr) & IPV6_ADDR_LINKLOCAL)
                        sin6.sin6_scope_id = ifa->idev->dev->ifindex;
                svc_age_temp_xprts_now(nn->nfsd_serv, (struct sockaddr *)&sin6);
        }
        spin_unlock(&nfsd_notifier_lock);

out:
        return NOTIFY_DONE;
}

static struct notifier_block nfsd_inet6addr_notifier = {
        .notifier_call = nfsd_inet6addr_event,
};
#endif

/* Only used under nfsd_mutex, so this atomic may be overkill: */
static atomic_t nfsd_notifier_refcount = ATOMIC_INIT(0);

/**
 * nfsd_destroy_serv - tear down NFSD's svc_serv for a namespace
 * @net: network namespace the NFS service is associated with
 */
void nfsd_destroy_serv(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv = nn->nfsd_serv;

        lockdep_assert_held(&nfsd_mutex);

        percpu_ref_kill_and_confirm(&nn->nfsd_serv_ref, nfsd_serv_done);
        wait_for_completion(&nn->nfsd_serv_confirm_done);
        wait_for_completion(&nn->nfsd_serv_free_done);
        /* percpu_ref_exit is called in nfsd_shutdown_net */

        spin_lock(&nfsd_notifier_lock);
        nn->nfsd_serv = NULL;
        spin_unlock(&nfsd_notifier_lock);

        /* check if the notifier still has clients */
        if (atomic_dec_return(&nfsd_notifier_refcount) == 0) {
                unregister_inetaddr_notifier(&nfsd_inetaddr_notifier);
#if IS_ENABLED(CONFIG_IPV6)
                unregister_inet6addr_notifier(&nfsd_inet6addr_notifier);
#endif
        }

        svc_xprt_destroy_all(serv, net);

        /*
         * write_ports can create the server without actually starting
         * any threads--if we get shut down before any threads are
         * started, then nfsd_destroy_serv will be run before any of this
         * other initialization has been done except the rpcb information.
         */
        svc_rpcb_cleanup(serv, net);

        nfsd_shutdown_net(net);
        svc_destroy(&serv);
}

void nfsd_reset_versions(struct nfsd_net *nn)
{
        int i;

        for (i = 0; i <= NFSD_MAXVERS; i++)
                if (nfsd_vers(nn, i, NFSD_TEST))
                        return;

        for (i = 0; i <= NFSD_MAXVERS; i++)
                if (i != 4)
                        nfsd_vers(nn, i, NFSD_SET);
                else {
                        int minor = 0;
                        while (nfsd_minorversion(nn, minor, NFSD_SET) >= 0)
                                minor++;
                }
}

/*
 * Each session guarantees a negotiated per slot memory cache for replies
 * which in turn consumes memory beyond the v2/v3/v4.0 server. A dedicated
 * NFSv4.1 server might want to use more memory for a DRC than a machine
 * with mutiple services.
 *
 * Impose a hard limit on the number of pages for the DRC which varies
 * according to the machines free pages. This is of course only a default.
 *
 * For now this is a #defined shift which could be under admin control
 * in the future.
 */
static void set_max_drc(void)
{
        #define NFSD_DRC_SIZE_SHIFT        7
        nfsd_drc_max_mem = (nr_free_buffer_pages()
                                        >> NFSD_DRC_SIZE_SHIFT) * PAGE_SIZE;
        nfsd_drc_mem_used = 0;
        dprintk("%s nfsd_drc_max_mem %lu \n", __func__, nfsd_drc_max_mem);
}

static int nfsd_get_default_max_blksize(void)
{
        struct sysinfo i;
        unsigned long long target;
        unsigned long ret;

        si_meminfo(&i);
        target = (i.totalram - i.totalhigh) << PAGE_SHIFT;
        /*
         * Aim for 1/4096 of memory per thread This gives 1MB on 4Gig
         * machines, but only uses 32K on 128M machines.  Bottom out at
         * 8K on 32M and smaller.  Of course, this is only a default.
         */
        target >>= 12;

        ret = NFSSVC_MAXBLKSIZE;
        while (ret > target && ret >= 8*1024*2)
                ret /= 2;
        return ret;
}

void nfsd_shutdown_threads(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv;

        mutex_lock(&nfsd_mutex);
        serv = nn->nfsd_serv;
        if (serv == NULL) {
                mutex_unlock(&nfsd_mutex);
                return;
        }

        /* Kill outstanding nfsd threads */
        svc_set_num_threads(serv, NULL, 0);
        nfsd_destroy_serv(net);
        mutex_unlock(&nfsd_mutex);
}

struct svc_rqst *nfsd_current_rqst(void)
{
        if (kthread_func(current) == nfsd)
                return kthread_data(current);
        return NULL;
}

int nfsd_create_serv(struct net *net)
{
        int error;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv;

        WARN_ON(!mutex_is_locked(&nfsd_mutex));
        if (nn->nfsd_serv)
                return 0;

        error = percpu_ref_init(&nn->nfsd_serv_ref, nfsd_serv_free,
                                0, GFP_KERNEL);
        if (error)
                return error;
        init_completion(&nn->nfsd_serv_free_done);
        init_completion(&nn->nfsd_serv_confirm_done);

        if (nfsd_max_blksize == 0)
                nfsd_max_blksize = nfsd_get_default_max_blksize();
        nfsd_reset_versions(nn);
        serv = svc_create_pooled(nfsd_programs, ARRAY_SIZE(nfsd_programs),
                                 &nn->nfsd_svcstats,
                                 nfsd_max_blksize, nfsd);
        if (serv == NULL)
                return -ENOMEM;

        serv->sv_maxconn = nn->max_connections;
        error = svc_bind(serv, net);
        if (error < 0) {
                svc_destroy(&serv);
                return error;
        }
        spin_lock(&nfsd_notifier_lock);
        nn->nfsd_serv = serv;
        spin_unlock(&nfsd_notifier_lock);

        set_max_drc();
        /* check if the notifier is already set */
        if (atomic_inc_return(&nfsd_notifier_refcount) == 1) {
                register_inetaddr_notifier(&nfsd_inetaddr_notifier);
#if IS_ENABLED(CONFIG_IPV6)
                register_inet6addr_notifier(&nfsd_inet6addr_notifier);
#endif
        }
        nfsd_reset_write_verifier(nn);
        return 0;
}

int nfsd_nrpools(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        if (nn->nfsd_serv == NULL)
                return 0;
        else
                return nn->nfsd_serv->sv_nrpools;
}

int nfsd_get_nrthreads(int n, int *nthreads, struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv = nn->nfsd_serv;
        int i;

        if (serv)
                for (i = 0; i < serv->sv_nrpools && i < n; i++)
                        nthreads[i] = serv->sv_pools[i].sp_nrthreads;
        return 0;
}

/**
 * nfsd_set_nrthreads - set the number of running threads in the net's service
 * @n: number of array members in @nthreads
 * @nthreads: array of thread counts for each pool
 * @net: network namespace to operate within
 *
 * This function alters the number of running threads for the given network
 * namespace in each pool. If passed an array longer then the number of pools
 * the extra pool settings are ignored. If passed an array shorter than the
 * number of pools, the missing values are interpreted as 0's.
 *
 * Returns 0 on success or a negative errno on error.
 */
int nfsd_set_nrthreads(int n, int *nthreads, struct net *net)
{
        int i = 0;
        int tot = 0;
        int err = 0;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        lockdep_assert_held(&nfsd_mutex);

        if (nn->nfsd_serv == NULL || n <= 0)
                return 0;

        /*
         * Special case: When n == 1, pass in NULL for the pool, so that the
         * change is distributed equally among them.
         */
        if (n == 1)
                return svc_set_num_threads(nn->nfsd_serv, NULL, nthreads[0]);

        if (n > nn->nfsd_serv->sv_nrpools)
                n = nn->nfsd_serv->sv_nrpools;

        /* enforce a global maximum number of threads */
        tot = 0;
        for (i = 0; i < n; i++) {
                nthreads[i] = min(nthreads[i], NFSD_MAXSERVS);
                tot += nthreads[i];
        }
        if (tot > NFSD_MAXSERVS) {
                /* total too large: scale down requested numbers */
                for (i = 0; i < n && tot > 0; i++) {
                        int new = nthreads[i] * NFSD_MAXSERVS / tot;
                        tot -= (nthreads[i] - new);
                        nthreads[i] = new;
                }
                for (i = 0; i < n && tot > 0; i++) {
                        nthreads[i]--;
                        tot--;
                }
        }

        /* apply the new numbers */
        for (i = 0; i < n; i++) {
                err = svc_set_num_threads(nn->nfsd_serv,
                                          &nn->nfsd_serv->sv_pools[i],
                                          nthreads[i]);
                if (err)
                        goto out;
        }

        /* Anything undefined in array is considered to be 0 */
        for (i = n; i < nn->nfsd_serv->sv_nrpools; ++i) {
                err = svc_set_num_threads(nn->nfsd_serv,
                                          &nn->nfsd_serv->sv_pools[i],
                                          0);
                if (err)
                        goto out;
        }
out:
        return err;
}

/**
 * nfsd_svc: start up or shut down the nfsd server
 * @n: number of array members in @nthreads
 * @nthreads: array of thread counts for each pool
 * @net: network namespace to operate within
 * @cred: credentials to use for xprt creation
 * @scope: server scope value (defaults to nodename)
 *
 * Adjust the number of threads in each pool and return the new
 * total number of threads in the service.
 */
int
nfsd_svc(int n, int *nthreads, struct net *net, const struct cred *cred, const char *scope)
{
        int        error;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv;

        lockdep_assert_held(&nfsd_mutex);

        dprintk("nfsd: creating service\n");

        strscpy(nn->nfsd_name, scope ? scope : utsname()->nodename,
                sizeof(nn->nfsd_name));

        error = nfsd_create_serv(net);
        if (error)
                goto out;
        serv = nn->nfsd_serv;

        error = nfsd_startup_net(net, cred);
        if (error)
                goto out_put;
        error = nfsd_set_nrthreads(n, nthreads, net);
        if (error)
                goto out_put;
        error = serv->sv_nrthreads;
out_put:
        if (serv->sv_nrthreads == 0)
                nfsd_destroy_serv(net);
out:
        return error;
}

#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
static bool
nfsd_support_acl_version(int vers)
{
        if (vers >= NFSD_ACL_MINVERS && vers < NFSD_ACL_NRVERS)
                return nfsd_acl_version[vers] != NULL;
        return false;
}

static int
nfsd_acl_rpcbind_set(struct net *net, const struct svc_program *progp,
                     u32 version, int family, unsigned short proto,
                     unsigned short port)
{
        if (!nfsd_support_acl_version(version) ||
            !nfsd_vers(net_generic(net, nfsd_net_id), version, NFSD_TEST))
                return 0;
        return svc_generic_rpcbind_set(net, progp, version, family,
                        proto, port);
}

static __be32
nfsd_acl_init_request(struct svc_rqst *rqstp,
                      const struct svc_program *progp,
                      struct svc_process_info *ret)
{
        struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
        int i;

        if (likely(nfsd_support_acl_version(rqstp->rq_vers) &&
            nfsd_vers(nn, rqstp->rq_vers, NFSD_TEST)))
                return svc_generic_init_request(rqstp, progp, ret);

        ret->mismatch.lovers = NFSD_ACL_NRVERS;
        for (i = NFSD_ACL_MINVERS; i < NFSD_ACL_NRVERS; i++) {
                if (nfsd_support_acl_version(rqstp->rq_vers) &&
                    nfsd_vers(nn, i, NFSD_TEST)) {
                        ret->mismatch.lovers = i;
                        break;
                }
        }
        if (ret->mismatch.lovers == NFSD_ACL_NRVERS)
                return rpc_prog_unavail;
        ret->mismatch.hivers = NFSD_ACL_MINVERS;
        for (i = NFSD_ACL_NRVERS - 1; i >= NFSD_ACL_MINVERS; i--) {
                if (nfsd_support_acl_version(rqstp->rq_vers) &&
                    nfsd_vers(nn, i, NFSD_TEST)) {
                        ret->mismatch.hivers = i;
                        break;
                }
        }
        return rpc_prog_mismatch;
}
#endif

static int
nfsd_rpcbind_set(struct net *net, const struct svc_program *progp,
                 u32 version, int family, unsigned short proto,
                 unsigned short port)
{
        if (!nfsd_vers(net_generic(net, nfsd_net_id), version, NFSD_TEST))
                return 0;
        return svc_generic_rpcbind_set(net, progp, version, family,
                        proto, port);
}

static __be32
nfsd_init_request(struct svc_rqst *rqstp,
                  const struct svc_program *progp,
                  struct svc_process_info *ret)
{
        struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
        int i;

        if (likely(nfsd_vers(nn, rqstp->rq_vers, NFSD_TEST)))
                return svc_generic_init_request(rqstp, progp, ret);

        ret->mismatch.lovers = NFSD_MAXVERS + 1;
        for (i = NFSD_MINVERS; i <= NFSD_MAXVERS; i++) {
                if (nfsd_vers(nn, i, NFSD_TEST)) {
                        ret->mismatch.lovers = i;
                        break;
                }
        }
        if (ret->mismatch.lovers > NFSD_MAXVERS)
                return rpc_prog_unavail;
        ret->mismatch.hivers = NFSD_MINVERS;
        for (i = NFSD_MAXVERS; i >= NFSD_MINVERS; i--) {
                if (nfsd_vers(nn, i, NFSD_TEST)) {
                        ret->mismatch.hivers = i;
                        break;
                }
        }
        return rpc_prog_mismatch;
}

/*
 * This is the NFS server kernel thread
 */
static int
nfsd(void *vrqstp)
{
        struct svc_rqst *rqstp = (struct svc_rqst *) vrqstp;
        struct svc_xprt *perm_sock = list_entry(rqstp->rq_server->sv_permsocks.next, typeof(struct svc_xprt), xpt_list);
        struct net *net = perm_sock->xpt_net;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        /* At this point, the thread shares current->fs
         * with the init process. We need to create files with the
         * umask as defined by the client instead of init's umask.
         */
        svc_thread_init_status(rqstp, unshare_fs_struct());

        current->fs->umask = 0;

        atomic_inc(&nfsd_th_cnt);

        set_freezable();

        /*
         * The main request loop
         */
        while (!svc_thread_should_stop(rqstp)) {
                /* Update sv_maxconn if it has changed */
                rqstp->rq_server->sv_maxconn = nn->max_connections;

                svc_recv(rqstp);

                nfsd_file_net_dispose(nn);
        }

        atomic_dec(&nfsd_th_cnt);

        /* Release the thread */
        svc_exit_thread(rqstp);
        return 0;
}

/**
 * nfsd_dispatch - Process an NFS or NFSACL or LOCALIO Request
 * @rqstp: incoming request
 *
 * This RPC dispatcher integrates the NFS server's duplicate reply cache.
 *
 * Return values:
 *  %0: Processing complete; do not send a Reply
 *  %1: Processing complete; send Reply in rqstp->rq_res
 */
int nfsd_dispatch(struct svc_rqst *rqstp)
{
        const struct svc_procedure *proc = rqstp->rq_procinfo;
        __be32 *statp = rqstp->rq_accept_statp;
        struct nfsd_cacherep *rp;
        unsigned int start, len;
        __be32 *nfs_reply;

        /*
         * Give the xdr decoder a chance to change this if it wants
         * (necessary in the NFSv4.0 compound case)
         */
        rqstp->rq_cachetype = proc->pc_cachetype;

        /*
         * ->pc_decode advances the argument stream past the NFS
         * Call header, so grab the header's starting location and
         * size now for the call to nfsd_cache_lookup().
         */
        start = xdr_stream_pos(&rqstp->rq_arg_stream);
        len = xdr_stream_remaining(&rqstp->rq_arg_stream);
        if (!proc->pc_decode(rqstp, &rqstp->rq_arg_stream))
                goto out_decode_err;

        /*
         * Release rq_status_counter setting it to an odd value after the rpc
         * request has been properly parsed. rq_status_counter is used to
         * notify the consumers if the rqstp fields are stable
         * (rq_status_counter is odd) or not meaningful (rq_status_counter
         * is even).
         */
        smp_store_release(&rqstp->rq_status_counter, rqstp->rq_status_counter | 1);

        rp = NULL;
        switch (nfsd_cache_lookup(rqstp, start, len, &rp)) {
        case RC_DOIT:
                break;
        case RC_REPLY:
                goto out_cached_reply;
        case RC_DROPIT:
                goto out_dropit;
        }

        nfs_reply = xdr_inline_decode(&rqstp->rq_res_stream, 0);
        *statp = proc->pc_func(rqstp);
        if (test_bit(RQ_DROPME, &rqstp->rq_flags))
                goto out_update_drop;

        if (!proc->pc_encode(rqstp, &rqstp->rq_res_stream))
                goto out_encode_err;

        /*
         * Release rq_status_counter setting it to an even value after the rpc
         * request has been properly processed.
         */
        smp_store_release(&rqstp->rq_status_counter, rqstp->rq_status_counter + 1);

        nfsd_cache_update(rqstp, rp, rqstp->rq_cachetype, nfs_reply);
out_cached_reply:
        return 1;

out_decode_err:
        trace_nfsd_garbage_args_err(rqstp);
        *statp = rpc_garbage_args;
        return 1;

out_update_drop:
        nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL);
out_dropit:
        return 0;

out_encode_err:
        trace_nfsd_cant_encode_err(rqstp);
        nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL);
        *statp = rpc_system_err;
        return 1;
}

/**
 * nfssvc_decode_voidarg - Decode void arguments
 * @rqstp: Server RPC transaction context
 * @xdr: XDR stream positioned at arguments to decode
 *
 * Return values:
 *   %false: Arguments were not valid
 *   %true: Decoding was successful
 */
bool nfssvc_decode_voidarg(struct svc_rqst *rqstp, struct xdr_stream *xdr)
{
        return true;
}

/**
 * nfssvc_encode_voidres - Encode void results
 * @rqstp: Server RPC transaction context
 * @xdr: XDR stream into which to encode results
 *
 * Return values:
 *   %false: Local error while encoding
 *   %true: Encoding was successful
 */
bool nfssvc_encode_voidres(struct svc_rqst *rqstp, struct xdr_stream *xdr)
{
        return true;
}



















































































































































































































































































































































































































































































































































































































































   68 

   68 






























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/act_pedit.c        Generic packet editor
 *
 * Authors:        Jamal Hadi Salim (2002-4)
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/slab.h>
#include <net/ipv6.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <linux/tc_act/tc_pedit.h>
#include <net/tc_act/tc_pedit.h>
#include <uapi/linux/tc_act/tc_pedit.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>

static struct tc_action_ops act_pedit_ops;

static const struct nla_policy pedit_policy[TCA_PEDIT_MAX + 1] = {
        [TCA_PEDIT_PARMS]        = { .len = sizeof(struct tc_pedit) },
        [TCA_PEDIT_PARMS_EX]        = { .len = sizeof(struct tc_pedit) },
        [TCA_PEDIT_KEYS_EX]   = { .type = NLA_NESTED },
};

static const struct nla_policy pedit_key_ex_policy[TCA_PEDIT_KEY_EX_MAX + 1] = {
        [TCA_PEDIT_KEY_EX_HTYPE] =
                NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_HDR_TYPE_MAX),
        [TCA_PEDIT_KEY_EX_CMD] = NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_CMD_MAX),
};

static struct tcf_pedit_key_ex *tcf_pedit_keys_ex_parse(struct nlattr *nla,
                                                        u8 n, struct netlink_ext_ack *extack)
{
        struct tcf_pedit_key_ex *keys_ex;
        struct tcf_pedit_key_ex *k;
        const struct nlattr *ka;
        int err = -EINVAL;
        int rem;

        if (!nla)
                return NULL;

        keys_ex = kcalloc(n, sizeof(*k), GFP_KERNEL);
        if (!keys_ex)
                return ERR_PTR(-ENOMEM);

        k = keys_ex;

        nla_for_each_nested(ka, nla, rem) {
                struct nlattr *tb[TCA_PEDIT_KEY_EX_MAX + 1];

                if (!n) {
                        NL_SET_ERR_MSG_MOD(extack, "Can't parse more extended keys than requested");
                        err = -EINVAL;
                        goto err_out;
                }
                n--;

                if (nla_type(ka) != TCA_PEDIT_KEY_EX) {
                        NL_SET_ERR_MSG_ATTR(extack, ka, "Unknown attribute, expected extended key");
                        err = -EINVAL;
                        goto err_out;
                }

                err = nla_parse_nested_deprecated(tb, TCA_PEDIT_KEY_EX_MAX,
                                                  ka, pedit_key_ex_policy,
                                                  NULL);
                if (err)
                        goto err_out;

                if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_HTYPE)) {
                        NL_SET_ERR_MSG(extack, "Missing required attribute");
                        err = -EINVAL;
                        goto err_out;
                }

                if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_CMD)) {
                        NL_SET_ERR_MSG(extack, "Missing required attribute");
                        err = -EINVAL;
                        goto err_out;
                }

                k->htype = nla_get_u16(tb[TCA_PEDIT_KEY_EX_HTYPE]);
                k->cmd = nla_get_u16(tb[TCA_PEDIT_KEY_EX_CMD]);

                k++;
        }

        if (n) {
                NL_SET_ERR_MSG_MOD(extack, "Not enough extended keys to parse");
                err = -EINVAL;
                goto err_out;
        }

        return keys_ex;

err_out:
        kfree(keys_ex);
        return ERR_PTR(err);
}

static int tcf_pedit_key_ex_dump(struct sk_buff *skb,
                                 struct tcf_pedit_key_ex *keys_ex, int n)
{
        struct nlattr *keys_start = nla_nest_start_noflag(skb,
                                                          TCA_PEDIT_KEYS_EX);

        if (!keys_start)
                goto nla_failure;
        for (; n > 0; n--) {
                struct nlattr *key_start;

                key_start = nla_nest_start_noflag(skb, TCA_PEDIT_KEY_EX);
                if (!key_start)
                        goto nla_failure;

                if (nla_put_u16(skb, TCA_PEDIT_KEY_EX_HTYPE, keys_ex->htype) ||
                    nla_put_u16(skb, TCA_PEDIT_KEY_EX_CMD, keys_ex->cmd))
                        goto nla_failure;

                nla_nest_end(skb, key_start);

                keys_ex++;
        }

        nla_nest_end(skb, keys_start);

        return 0;
nla_failure:
        nla_nest_cancel(skb, keys_start);
        return -EINVAL;
}

static void tcf_pedit_cleanup_rcu(struct rcu_head *head)
{
        struct tcf_pedit_parms *parms =
                container_of(head, struct tcf_pedit_parms, rcu);

        kfree(parms->tcfp_keys_ex);
        kfree(parms->tcfp_keys);

        kfree(parms);
}

static int tcf_pedit_init(struct net *net, struct nlattr *nla,
                          struct nlattr *est, struct tc_action **a,
                          struct tcf_proto *tp, u32 flags,
                          struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id);
        bool bind = flags & TCA_ACT_FLAGS_BIND;
        struct tcf_chain *goto_ch = NULL;
        struct tcf_pedit_parms *oparms, *nparms;
        struct nlattr *tb[TCA_PEDIT_MAX + 1];
        struct tc_pedit *parm;
        struct nlattr *pattr;
        struct tcf_pedit *p;
        int ret = 0, err;
        int i, ksize;
        u32 index;

        if (!nla) {
                NL_SET_ERR_MSG_MOD(extack, "Pedit requires attributes to be passed");
                return -EINVAL;
        }

        err = nla_parse_nested_deprecated(tb, TCA_PEDIT_MAX, nla,
                                          pedit_policy, NULL);
        if (err < 0)
                return err;

        pattr = tb[TCA_PEDIT_PARMS];
        if (!pattr)
                pattr = tb[TCA_PEDIT_PARMS_EX];
        if (!pattr) {
                NL_SET_ERR_MSG_MOD(extack, "Missing required TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute");
                return -EINVAL;
        }

        parm = nla_data(pattr);

        index = parm->index;
        err = tcf_idr_check_alloc(tn, &index, a, bind);
        if (!err) {
                ret = tcf_idr_create_from_flags(tn, index, est, a,
                                                &act_pedit_ops, bind, flags);
                if (ret) {
                        tcf_idr_cleanup(tn, index);
                        return ret;
                }
                ret = ACT_P_CREATED;
        } else if (err > 0) {
                if (bind)
                        return ACT_P_BOUND;
                if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
                        ret = -EEXIST;
                        goto out_release;
                }
        } else {
                return err;
        }

        if (!parm->nkeys) {
                NL_SET_ERR_MSG_MOD(extack, "Pedit requires keys to be passed");
                ret = -EINVAL;
                goto out_release;
        }
        ksize = parm->nkeys * sizeof(struct tc_pedit_key);
        if (nla_len(pattr) < sizeof(*parm) + ksize) {
                NL_SET_ERR_MSG_ATTR(extack, pattr, "Length of TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute is invalid");
                ret = -EINVAL;
                goto out_release;
        }

        nparms = kzalloc(sizeof(*nparms), GFP_KERNEL);
        if (!nparms) {
                ret = -ENOMEM;
                goto out_release;
        }

        nparms->tcfp_keys_ex =
                tcf_pedit_keys_ex_parse(tb[TCA_PEDIT_KEYS_EX], parm->nkeys, extack);
        if (IS_ERR(nparms->tcfp_keys_ex)) {
                ret = PTR_ERR(nparms->tcfp_keys_ex);
                goto out_free;
        }

        err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
        if (err < 0) {
                ret = err;
                goto out_free_ex;
        }

        nparms->tcfp_off_max_hint = 0;
        nparms->tcfp_flags = parm->flags;
        nparms->tcfp_nkeys = parm->nkeys;

        nparms->tcfp_keys = kmemdup(parm->keys, ksize, GFP_KERNEL);
        if (!nparms->tcfp_keys) {
                ret = -ENOMEM;
                goto put_chain;
        }

        for (i = 0; i < nparms->tcfp_nkeys; ++i) {
                u32 offmask = nparms->tcfp_keys[i].offmask;
                u32 cur = nparms->tcfp_keys[i].off;

                /* The AT option can be added to static offsets in the datapath */
                if (!offmask && cur % 4) {
                        NL_SET_ERR_MSG_MOD(extack, "Offsets must be on 32bit boundaries");
                        ret = -EINVAL;
                        goto out_free_keys;
                }

                /* sanitize the shift value for any later use */
                nparms->tcfp_keys[i].shift = min_t(size_t,
                                                   BITS_PER_TYPE(int) - 1,
                                                   nparms->tcfp_keys[i].shift);

                /* The AT option can read a single byte, we can bound the actual
                 * value with uchar max.
                 */
                cur += (0xff & offmask) >> nparms->tcfp_keys[i].shift;

                /* Each key touches 4 bytes starting from the computed offset */
                nparms->tcfp_off_max_hint =
                        max(nparms->tcfp_off_max_hint, cur + 4);
        }

        p = to_pedit(*a);

        spin_lock_bh(&p->tcf_lock);
        goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
        oparms = rcu_replace_pointer(p->parms, nparms, 1);
        spin_unlock_bh(&p->tcf_lock);

        if (oparms)
                call_rcu(&oparms->rcu, tcf_pedit_cleanup_rcu);

        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);

        return ret;

out_free_keys:
        kfree(nparms->tcfp_keys);
put_chain:
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
out_free_ex:
        kfree(nparms->tcfp_keys_ex);
out_free:
        kfree(nparms);
out_release:
        tcf_idr_release(*a, bind);
        return ret;
}

static void tcf_pedit_cleanup(struct tc_action *a)
{
        struct tcf_pedit *p = to_pedit(a);
        struct tcf_pedit_parms *parms;

        parms = rcu_dereference_protected(p->parms, 1);

        if (parms)
                call_rcu(&parms->rcu, tcf_pedit_cleanup_rcu);
}

static bool offset_valid(struct sk_buff *skb, int offset)
{
        if (offset > 0 && offset > skb->len)
                return false;

        if  (offset < 0 && -offset > skb_headroom(skb))
                return false;

        return true;
}

static int pedit_l4_skb_offset(struct sk_buff *skb, int *hoffset, const int header_type)
{
        const int noff = skb_network_offset(skb);
        int ret = -EINVAL;
        struct iphdr _iph;

        switch (skb->protocol) {
        case htons(ETH_P_IP): {
                const struct iphdr *iph = skb_header_pointer(skb, noff, sizeof(_iph), &_iph);

                if (!iph)
                        goto out;
                *hoffset = noff + iph->ihl * 4;
                ret = 0;
                break;
        }
        case htons(ETH_P_IPV6):
                ret = ipv6_find_hdr(skb, hoffset, header_type, NULL, NULL) == header_type ? 0 : -EINVAL;
                break;
        }
out:
        return ret;
}

static int pedit_skb_hdr_offset(struct sk_buff *skb,
                                 enum pedit_header_type htype, int *hoffset)
{
        int ret = -EINVAL;
        /* 'htype' is validated in the netlink parsing */
        switch (htype) {
        case TCA_PEDIT_KEY_EX_HDR_TYPE_ETH:
                if (skb_mac_header_was_set(skb)) {
                        *hoffset = skb_mac_offset(skb);
                        ret = 0;
                }
                break;
        case TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK:
        case TCA_PEDIT_KEY_EX_HDR_TYPE_IP4:
        case TCA_PEDIT_KEY_EX_HDR_TYPE_IP6:
                *hoffset = skb_network_offset(skb);
                ret = 0;
                break;
        case TCA_PEDIT_KEY_EX_HDR_TYPE_TCP:
                ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_TCP);
                break;
        case TCA_PEDIT_KEY_EX_HDR_TYPE_UDP:
                ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_UDP);
                break;
        default:
                break;
        }
        return ret;
}

TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb,
                                    const struct tc_action *a,
                                    struct tcf_result *res)
{
        enum pedit_header_type htype = TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK;
        enum pedit_cmd cmd = TCA_PEDIT_KEY_EX_CMD_SET;
        struct tcf_pedit *p = to_pedit(a);
        struct tcf_pedit_key_ex *tkey_ex;
        struct tcf_pedit_parms *parms;
        struct tc_pedit_key *tkey;
        u32 max_offset;
        int i;

        parms = rcu_dereference_bh(p->parms);

        max_offset = (skb_transport_header_was_set(skb) ?
                      skb_transport_offset(skb) :
                      skb_network_offset(skb)) +
                     parms->tcfp_off_max_hint;
        if (skb_ensure_writable(skb, min(skb->len, max_offset)))
                goto done;

        tcf_lastuse_update(&p->tcf_tm);
        tcf_action_update_bstats(&p->common, skb);

        tkey = parms->tcfp_keys;
        tkey_ex = parms->tcfp_keys_ex;

        for (i = parms->tcfp_nkeys; i > 0; i--, tkey++) {
                int offset = tkey->off;
                int hoffset = 0;
                u32 *ptr, hdata;
                u32 val;
                int rc;

                if (tkey_ex) {
                        htype = tkey_ex->htype;
                        cmd = tkey_ex->cmd;

                        tkey_ex++;
                }

                rc = pedit_skb_hdr_offset(skb, htype, &hoffset);
                if (rc) {
                        pr_info_ratelimited("tc action pedit unable to extract header offset for header type (0x%x)\n", htype);
                        goto bad;
                }

                if (tkey->offmask) {
                        u8 *d, _d;

                        if (!offset_valid(skb, hoffset + tkey->at)) {
                                pr_info_ratelimited("tc action pedit 'at' offset %d out of bounds\n",
                                                    hoffset + tkey->at);
                                goto bad;
                        }
                        d = skb_header_pointer(skb, hoffset + tkey->at,
                                               sizeof(_d), &_d);
                        if (!d)
                                goto bad;

                        offset += (*d & tkey->offmask) >> tkey->shift;
                        if (offset % 4) {
                                pr_info_ratelimited("tc action pedit offset must be on 32 bit boundaries\n");
                                goto bad;
                        }
                }

                if (!offset_valid(skb, hoffset + offset)) {
                        pr_info_ratelimited("tc action pedit offset %d out of bounds\n", hoffset + offset);
                        goto bad;
                }

                ptr = skb_header_pointer(skb, hoffset + offset,
                                         sizeof(hdata), &hdata);
                if (!ptr)
                        goto bad;
                /* just do it, baby */
                switch (cmd) {
                case TCA_PEDIT_KEY_EX_CMD_SET:
                        val = tkey->val;
                        break;
                case TCA_PEDIT_KEY_EX_CMD_ADD:
                        val = (*ptr + tkey->val) & ~tkey->mask;
                        break;
                default:
                        pr_info_ratelimited("tc action pedit bad command (%d)\n", cmd);
                        goto bad;
                }

                *ptr = ((*ptr & tkey->mask) ^ val);
                if (ptr == &hdata)
                        skb_store_bits(skb, hoffset + offset, ptr, 4);
        }

        goto done;

bad:
        tcf_action_inc_overlimit_qstats(&p->common);
done:
        return p->tcf_action;
}

static void tcf_pedit_stats_update(struct tc_action *a, u64 bytes, u64 packets,
                                   u64 drops, u64 lastuse, bool hw)
{
        struct tcf_pedit *d = to_pedit(a);
        struct tcf_t *tm = &d->tcf_tm;

        tcf_action_update_stats(a, bytes, packets, drops, hw);
        tm->lastuse = max_t(u64, tm->lastuse, lastuse);
}

static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a,
                          int bind, int ref)
{
        unsigned char *b = skb_tail_pointer(skb);
        struct tcf_pedit *p = to_pedit(a);
        struct tcf_pedit_parms *parms;
        struct tc_pedit *opt;
        struct tcf_t t;
        int s;

        spin_lock_bh(&p->tcf_lock);
        parms = rcu_dereference_protected(p->parms, 1);
        s = struct_size(opt, keys, parms->tcfp_nkeys);

        opt = kzalloc(s, GFP_ATOMIC);
        if (unlikely(!opt)) {
                spin_unlock_bh(&p->tcf_lock);
                return -ENOBUFS;
        }
        opt->nkeys = parms->tcfp_nkeys;

        memcpy(opt->keys, parms->tcfp_keys,
               flex_array_size(opt, keys, parms->tcfp_nkeys));
        opt->index = p->tcf_index;
        opt->flags = parms->tcfp_flags;
        opt->action = p->tcf_action;
        opt->refcnt = refcount_read(&p->tcf_refcnt) - ref;
        opt->bindcnt = atomic_read(&p->tcf_bindcnt) - bind;

        if (parms->tcfp_keys_ex) {
                if (tcf_pedit_key_ex_dump(skb, parms->tcfp_keys_ex,
                                          parms->tcfp_nkeys))
                        goto nla_put_failure;

                if (nla_put(skb, TCA_PEDIT_PARMS_EX, s, opt))
                        goto nla_put_failure;
        } else {
                if (nla_put(skb, TCA_PEDIT_PARMS, s, opt))
                        goto nla_put_failure;
        }

        tcf_tm_dump(&t, &p->tcf_tm);
        if (nla_put_64bit(skb, TCA_PEDIT_TM, sizeof(t), &t, TCA_PEDIT_PAD))
                goto nla_put_failure;
        spin_unlock_bh(&p->tcf_lock);

        kfree(opt);
        return skb->len;

nla_put_failure:
        spin_unlock_bh(&p->tcf_lock);
        nlmsg_trim(skb, b);
        kfree(opt);
        return -1;
}

static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data,
                                       u32 *index_inc, bool bind,
                                       struct netlink_ext_ack *extack)
{
        if (bind) {
                struct flow_action_entry *entry = entry_data;
                int k;

                for (k = 0; k < tcf_pedit_nkeys(act); k++) {
                        switch (tcf_pedit_cmd(act, k)) {
                        case TCA_PEDIT_KEY_EX_CMD_SET:
                                entry->id = FLOW_ACTION_MANGLE;
                                break;
                        case TCA_PEDIT_KEY_EX_CMD_ADD:
                                entry->id = FLOW_ACTION_ADD;
                                break;
                        default:
                                NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload");
                                return -EOPNOTSUPP;
                        }
                        entry->mangle.htype = tcf_pedit_htype(act, k);
                        entry->mangle.mask = tcf_pedit_mask(act, k);
                        entry->mangle.val = tcf_pedit_val(act, k);
                        entry->mangle.offset = tcf_pedit_offset(act, k);
                        entry->hw_stats = tc_act_hw_stats(act->hw_stats);
                        entry++;
                }
                *index_inc = k;
        } else {
                struct flow_offload_action *fl_action = entry_data;
                u32 cmd = tcf_pedit_cmd(act, 0);
                int k;

                switch (cmd) {
                case TCA_PEDIT_KEY_EX_CMD_SET:
                        fl_action->id = FLOW_ACTION_MANGLE;
                        break;
                case TCA_PEDIT_KEY_EX_CMD_ADD:
                        fl_action->id = FLOW_ACTION_ADD;
                        break;
                default:
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload");
                        return -EOPNOTSUPP;
                }

                for (k = 1; k < tcf_pedit_nkeys(act); k++) {
                        if (cmd != tcf_pedit_cmd(act, k)) {
                                NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload");
                                return -EOPNOTSUPP;
                        }
                }
        }

        return 0;
}

static struct tc_action_ops act_pedit_ops = {
        .kind                =        "pedit",
        .id                =        TCA_ID_PEDIT,
        .owner                =        THIS_MODULE,
        .act                =        tcf_pedit_act,
        .stats_update        =        tcf_pedit_stats_update,
        .dump                =        tcf_pedit_dump,
        .cleanup        =        tcf_pedit_cleanup,
        .init                =        tcf_pedit_init,
        .offload_act_setup =        tcf_pedit_offload_act_setup,
        .size                =        sizeof(struct tcf_pedit),
};
MODULE_ALIAS_NET_ACT("pedit");

static __net_init int pedit_init_net(struct net *net)
{
        struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id);

        return tc_action_net_init(net, tn, &act_pedit_ops);
}

static void __net_exit pedit_exit_net(struct list_head *net_list)
{
        tc_action_net_exit(net_list, act_pedit_ops.net_id);
}

static struct pernet_operations pedit_net_ops = {
        .init = pedit_init_net,
        .exit_batch = pedit_exit_net,
        .id   = &act_pedit_ops.net_id,
        .size = sizeof(struct tc_action_net),
};

MODULE_AUTHOR("Jamal Hadi Salim(2002-4)");
MODULE_DESCRIPTION("Generic Packet Editor actions");
MODULE_LICENSE("GPL");

static int __init pedit_init_module(void)
{
        return tcf_register_action(&act_pedit_ops, &pedit_net_ops);
}

static void __exit pedit_cleanup_module(void)
{
        tcf_unregister_action(&act_pedit_ops, &pedit_net_ops);
}

module_init(pedit_init_module);
module_exit(pedit_cleanup_module);















































































   37 








   23 



























































































   35 

















































































   35 







   35 

   35 
   35 



   35 








































































































   22 









   21 




   22 
























   22 
   22 
   22 



   22 
   22 





































































































































































































































































   68 






   68 












   68 



   68 

















   24 






   16 
   16 






   10 
   10 






   19 






   37 




   37 


   37 







    2 












    1 








































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
/*
 * net/tipc/bcast.c: TIPC broadcast code
 *
 * Copyright (c) 2004-2006, 2014-2017, Ericsson AB
 * Copyright (c) 2004, Intel Corporation.
 * Copyright (c) 2005, 2010-2011, Wind River Systems
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the names of the copyright holders nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <linux/tipc_config.h>
#include "socket.h"
#include "msg.h"
#include "bcast.h"
#include "link.h"
#include "name_table.h"

#define BCLINK_WIN_DEFAULT  50        /* bcast link window size (default) */
#define BCLINK_WIN_MIN      32        /* bcast minimum link window size */

const char tipc_bclink_name[] = "broadcast-link";
unsigned long sysctl_tipc_bc_retruni __read_mostly;

/**
 * struct tipc_bc_base - base structure for keeping broadcast send state
 * @link: broadcast send link structure
 * @inputq: data input queue; will only carry SOCK_WAKEUP messages
 * @dests: array keeping number of reachable destinations per bearer
 * @primary_bearer: a bearer having links to all broadcast destinations, if any
 * @bcast_support: indicates if primary bearer, if any, supports broadcast
 * @force_bcast: forces broadcast for multicast traffic
 * @rcast_support: indicates if all peer nodes support replicast
 * @force_rcast: forces replicast for multicast traffic
 * @rc_ratio: dest count as percentage of cluster size where send method changes
 * @bc_threshold: calculated from rc_ratio; if dests > threshold use broadcast
 */
struct tipc_bc_base {
        struct tipc_link *link;
        struct sk_buff_head inputq;
        int dests[MAX_BEARERS];
        int primary_bearer;
        bool bcast_support;
        bool force_bcast;
        bool rcast_support;
        bool force_rcast;
        int rc_ratio;
        int bc_threshold;
};

static struct tipc_bc_base *tipc_bc_base(struct net *net)
{
        return tipc_net(net)->bcbase;
}

/* tipc_bcast_get_mtu(): -get the MTU currently used by broadcast link
 * Note: the MTU is decremented to give room for a tunnel header, in
 * case the message needs to be sent as replicast
 */
int tipc_bcast_get_mtu(struct net *net)
{
        return tipc_link_mss(tipc_bc_sndlink(net));
}

void tipc_bcast_toggle_rcast(struct net *net, bool supp)
{
        tipc_bc_base(net)->rcast_support = supp;
}

static void tipc_bcbase_calc_bc_threshold(struct net *net)
{
        struct tipc_bc_base *bb = tipc_bc_base(net);
        int cluster_size = tipc_link_bc_peers(tipc_bc_sndlink(net));

        bb->bc_threshold = 1 + (cluster_size * bb->rc_ratio / 100);
}

/* tipc_bcbase_select_primary(): find a bearer with links to all destinations,
 *                               if any, and make it primary bearer
 */
static void tipc_bcbase_select_primary(struct net *net)
{
        struct tipc_bc_base *bb = tipc_bc_base(net);
        int all_dests =  tipc_link_bc_peers(bb->link);
        int max_win = tipc_link_max_win(bb->link);
        int min_win = tipc_link_min_win(bb->link);
        int i, mtu, prim;

        bb->primary_bearer = INVALID_BEARER_ID;
        bb->bcast_support = true;

        if (!all_dests)
                return;

        for (i = 0; i < MAX_BEARERS; i++) {
                if (!bb->dests[i])
                        continue;

                mtu = tipc_bearer_mtu(net, i);
                if (mtu < tipc_link_mtu(bb->link)) {
                        tipc_link_set_mtu(bb->link, mtu);
                        tipc_link_set_queue_limits(bb->link,
                                                   min_win,
                                                   max_win);
                }
                bb->bcast_support &= tipc_bearer_bcast_support(net, i);
                if (bb->dests[i] < all_dests)
                        continue;

                bb->primary_bearer = i;

                /* Reduce risk that all nodes select same primary */
                if ((i ^ tipc_own_addr(net)) & 1)
                        break;
        }
        prim = bb->primary_bearer;
        if (prim != INVALID_BEARER_ID)
                bb->bcast_support = tipc_bearer_bcast_support(net, prim);
}

void tipc_bcast_inc_bearer_dst_cnt(struct net *net, int bearer_id)
{
        struct tipc_bc_base *bb = tipc_bc_base(net);

        tipc_bcast_lock(net);
        bb->dests[bearer_id]++;
        tipc_bcbase_select_primary(net);
        tipc_bcast_unlock(net);
}

void tipc_bcast_dec_bearer_dst_cnt(struct net *net, int bearer_id)
{
        struct tipc_bc_base *bb = tipc_bc_base(net);

        tipc_bcast_lock(net);
        bb->dests[bearer_id]--;
        tipc_bcbase_select_primary(net);
        tipc_bcast_unlock(net);
}

/* tipc_bcbase_xmit - broadcast a packet queue across one or more bearers
 *
 * Note that number of reachable destinations, as indicated in the dests[]
 * array, may transitionally differ from the number of destinations indicated
 * in each sent buffer. We can sustain this. Excess destination nodes will
 * drop and never acknowledge the unexpected packets, and missing destinations
 * will either require retransmission (if they are just about to be added to
 * the bearer), or be removed from the buffer's 'ackers' counter (if they
 * just went down)
 */
static void tipc_bcbase_xmit(struct net *net, struct sk_buff_head *xmitq)
{
        int bearer_id;
        struct tipc_bc_base *bb = tipc_bc_base(net);
        struct sk_buff *skb, *_skb;
        struct sk_buff_head _xmitq;

        if (skb_queue_empty(xmitq))
                return;

        /* The typical case: at least one bearer has links to all nodes */
        bearer_id = bb->primary_bearer;
        if (bearer_id >= 0) {
                tipc_bearer_bc_xmit(net, bearer_id, xmitq);
                return;
        }

        /* We have to transmit across all bearers */
        __skb_queue_head_init(&_xmitq);
        for (bearer_id = 0; bearer_id < MAX_BEARERS; bearer_id++) {
                if (!bb->dests[bearer_id])
                        continue;

                skb_queue_walk(xmitq, skb) {
                        _skb = pskb_copy_for_clone(skb, GFP_ATOMIC);
                        if (!_skb)
                                break;
                        __skb_queue_tail(&_xmitq, _skb);
                }
                tipc_bearer_bc_xmit(net, bearer_id, &_xmitq);
        }
        __skb_queue_purge(xmitq);
        __skb_queue_purge(&_xmitq);
}

static void tipc_bcast_select_xmit_method(struct net *net, int dests,
                                          struct tipc_mc_method *method)
{
        struct tipc_bc_base *bb = tipc_bc_base(net);
        unsigned long exp = method->expires;

        /* Broadcast supported by used bearer/bearers? */
        if (!bb->bcast_support) {
                method->rcast = true;
                return;
        }
        /* Any destinations which don't support replicast ? */
        if (!bb->rcast_support) {
                method->rcast = false;
                return;
        }
        /* Can current method be changed ? */
        method->expires = jiffies + TIPC_METHOD_EXPIRE;
        if (method->mandatory)
                return;

        if (!(tipc_net(net)->capabilities & TIPC_MCAST_RBCTL) &&
            time_before(jiffies, exp))
                return;

        /* Configuration as force 'broadcast' method */
        if (bb->force_bcast) {
                method->rcast = false;
                return;
        }
        /* Configuration as force 'replicast' method */
        if (bb->force_rcast) {
                method->rcast = true;
                return;
        }
        /* Configuration as 'autoselect' or default method */
        /* Determine method to use now */
        method->rcast = dests <= bb->bc_threshold;
}

/* tipc_bcast_xmit - broadcast the buffer chain to all external nodes
 * @net: the applicable net namespace
 * @pkts: chain of buffers containing message
 * @cong_link_cnt: set to 1 if broadcast link is congested, otherwise 0
 * Consumes the buffer chain.
 * Returns 0 if success, otherwise errno: -EHOSTUNREACH,-EMSGSIZE
 */
int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts,
                    u16 *cong_link_cnt)
{
        struct tipc_link *l = tipc_bc_sndlink(net);
        struct sk_buff_head xmitq;
        int rc = 0;

        __skb_queue_head_init(&xmitq);
        tipc_bcast_lock(net);
        if (tipc_link_bc_peers(l))
                rc = tipc_link_xmit(l, pkts, &xmitq);
        tipc_bcast_unlock(net);
        tipc_bcbase_xmit(net, &xmitq);
        __skb_queue_purge(pkts);
        if (rc == -ELINKCONG) {
                *cong_link_cnt = 1;
                rc = 0;
        }
        return rc;
}

/* tipc_rcast_xmit - replicate and send a message to given destination nodes
 * @net: the applicable net namespace
 * @pkts: chain of buffers containing message
 * @dests: list of destination nodes
 * @cong_link_cnt: returns number of congested links
 * @cong_links: returns identities of congested links
 * Returns 0 if success, otherwise errno
 */
static int tipc_rcast_xmit(struct net *net, struct sk_buff_head *pkts,
                           struct tipc_nlist *dests, u16 *cong_link_cnt)
{
        struct tipc_dest *dst, *tmp;
        struct sk_buff_head _pkts;
        u32 dnode, selector;

        selector = msg_link_selector(buf_msg(skb_peek(pkts)));
        __skb_queue_head_init(&_pkts);

        list_for_each_entry_safe(dst, tmp, &dests->list, list) {
                dnode = dst->node;
                if (!tipc_msg_pskb_copy(dnode, pkts, &_pkts))
                        return -ENOMEM;

                /* Any other return value than -ELINKCONG is ignored */
                if (tipc_node_xmit(net, &_pkts, dnode, selector) == -ELINKCONG)
                        (*cong_link_cnt)++;
        }
        return 0;
}

/* tipc_mcast_send_sync - deliver a dummy message with SYN bit
 * @net: the applicable net namespace
 * @skb: socket buffer to copy
 * @method: send method to be used
 * @dests: destination nodes for message.
 * Returns 0 if success, otherwise errno
 */
static int tipc_mcast_send_sync(struct net *net, struct sk_buff *skb,
                                struct tipc_mc_method *method,
                                struct tipc_nlist *dests)
{
        struct tipc_msg *hdr, *_hdr;
        struct sk_buff_head tmpq;
        u16 cong_link_cnt = 0;
        struct sk_buff *_skb;
        int rc = 0;

        /* Is a cluster supporting with new capabilities ? */
        if (!(tipc_net(net)->capabilities & TIPC_MCAST_RBCTL))
                return 0;

        hdr = buf_msg(skb);
        if (msg_user(hdr) == MSG_FRAGMENTER)
                hdr = msg_inner_hdr(hdr);
        if (msg_type(hdr) != TIPC_MCAST_MSG)
                return 0;

        /* Allocate dummy message */
        _skb = tipc_buf_acquire(MCAST_H_SIZE, GFP_KERNEL);
        if (!_skb)
                return -ENOMEM;

        /* Preparing for 'synching' header */
        msg_set_syn(hdr, 1);

        /* Copy skb's header into a dummy header */
        skb_copy_to_linear_data(_skb, hdr, MCAST_H_SIZE);
        skb_orphan(_skb);

        /* Reverse method for dummy message */
        _hdr = buf_msg(_skb);
        msg_set_size(_hdr, MCAST_H_SIZE);
        msg_set_is_rcast(_hdr, !msg_is_rcast(hdr));
        msg_set_errcode(_hdr, TIPC_ERR_NO_PORT);

        __skb_queue_head_init(&tmpq);
        __skb_queue_tail(&tmpq, _skb);
        if (method->rcast)
                rc = tipc_bcast_xmit(net, &tmpq, &cong_link_cnt);
        else
                rc = tipc_rcast_xmit(net, &tmpq, dests, &cong_link_cnt);

        /* This queue should normally be empty by now */
        __skb_queue_purge(&tmpq);

        return rc;
}

/* tipc_mcast_xmit - deliver message to indicated destination nodes
 *                   and to identified node local sockets
 * @net: the applicable net namespace
 * @pkts: chain of buffers containing message
 * @method: send method to be used
 * @dests: destination nodes for message.
 * @cong_link_cnt: returns number of encountered congested destination links
 * Consumes buffer chain.
 * Returns 0 if success, otherwise errno
 */
int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts,
                    struct tipc_mc_method *method, struct tipc_nlist *dests,
                    u16 *cong_link_cnt)
{
        struct sk_buff_head inputq, localq;
        bool rcast = method->rcast;
        struct tipc_msg *hdr;
        struct sk_buff *skb;
        int rc = 0;

        skb_queue_head_init(&inputq);
        __skb_queue_head_init(&localq);

        /* Clone packets before they are consumed by next call */
        if (dests->local && !tipc_msg_reassemble(pkts, &localq)) {
                rc = -ENOMEM;
                goto exit;
        }
        /* Send according to determined transmit method */
        if (dests->remote) {
                tipc_bcast_select_xmit_method(net, dests->remote, method);

                skb = skb_peek(pkts);
                hdr = buf_msg(skb);
                if (msg_user(hdr) == MSG_FRAGMENTER)
                        hdr = msg_inner_hdr(hdr);
                msg_set_is_rcast(hdr, method->rcast);

                /* Switch method ? */
                if (rcast != method->rcast) {
                        rc = tipc_mcast_send_sync(net, skb, method, dests);
                        if (unlikely(rc)) {
                                pr_err("Unable to send SYN: method %d, rc %d\n",
                                       rcast, rc);
                                goto exit;
                        }
                }

                if (method->rcast)
                        rc = tipc_rcast_xmit(net, pkts, dests, cong_link_cnt);
                else
                        rc = tipc_bcast_xmit(net, pkts, cong_link_cnt);
        }

        if (dests->local) {
                tipc_loopback_trace(net, &localq);
                tipc_sk_mcast_rcv(net, &localq, &inputq);
        }
exit:
        /* This queue should normally be empty by now */
        __skb_queue_purge(pkts);
        return rc;
}

/* tipc_bcast_rcv - receive a broadcast packet, and deliver to rcv link
 *
 * RCU is locked, no other locks set
 */
int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb)
{
        struct tipc_msg *hdr = buf_msg(skb);
        struct sk_buff_head *inputq = &tipc_bc_base(net)->inputq;
        struct sk_buff_head xmitq;
        int rc;

        __skb_queue_head_init(&xmitq);

        if (msg_mc_netid(hdr) != tipc_netid(net) || !tipc_link_is_up(l)) {
                kfree_skb(skb);
                return 0;
        }

        tipc_bcast_lock(net);
        if (msg_user(hdr) == BCAST_PROTOCOL)
                rc = tipc_link_bc_nack_rcv(l, skb, &xmitq);
        else
                rc = tipc_link_rcv(l, skb, NULL);
        tipc_bcast_unlock(net);

        tipc_bcbase_xmit(net, &xmitq);

        /* Any socket wakeup messages ? */
        if (!skb_queue_empty(inputq))
                tipc_sk_rcv(net, inputq);

        return rc;
}

/* tipc_bcast_ack_rcv - receive and handle a broadcast acknowledge
 *
 * RCU is locked, no other locks set
 */
void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l,
                        struct tipc_msg *hdr)
{
        struct sk_buff_head *inputq = &tipc_bc_base(net)->inputq;
        u16 acked = msg_bcast_ack(hdr);
        struct sk_buff_head xmitq;

        /* Ignore bc acks sent by peer before bcast synch point was received */
        if (msg_bc_ack_invalid(hdr))
                return;

        __skb_queue_head_init(&xmitq);

        tipc_bcast_lock(net);
        tipc_link_bc_ack_rcv(l, acked, 0, NULL, &xmitq, NULL);
        tipc_bcast_unlock(net);

        tipc_bcbase_xmit(net, &xmitq);

        /* Any socket wakeup messages ? */
        if (!skb_queue_empty(inputq))
                tipc_sk_rcv(net, inputq);
}

/* tipc_bcast_synch_rcv -  check and update rcv link with peer's send state
 *
 * RCU is locked, no other locks set
 */
int tipc_bcast_sync_rcv(struct net *net, struct tipc_link *l,
                        struct tipc_msg *hdr,
                        struct sk_buff_head *retrq)
{
        struct sk_buff_head *inputq = &tipc_bc_base(net)->inputq;
        struct tipc_gap_ack_blks *ga;
        struct sk_buff_head xmitq;
        int rc = 0;

        __skb_queue_head_init(&xmitq);

        tipc_bcast_lock(net);
        if (msg_type(hdr) != STATE_MSG) {
                tipc_link_bc_init_rcv(l, hdr);
        } else if (!msg_bc_ack_invalid(hdr)) {
                tipc_get_gap_ack_blks(&ga, l, hdr, false);
                if (!sysctl_tipc_bc_retruni)
                        retrq = &xmitq;
                rc = tipc_link_bc_ack_rcv(l, msg_bcast_ack(hdr),
                                          msg_bc_gap(hdr), ga, &xmitq,
                                          retrq);
                rc |= tipc_link_bc_sync_rcv(l, hdr, &xmitq);
        }
        tipc_bcast_unlock(net);

        tipc_bcbase_xmit(net, &xmitq);

        /* Any socket wakeup messages ? */
        if (!skb_queue_empty(inputq))
                tipc_sk_rcv(net, inputq);
        return rc;
}

/* tipc_bcast_add_peer - add a peer node to broadcast link and bearer
 *
 * RCU is locked, node lock is set
 */
void tipc_bcast_add_peer(struct net *net, struct tipc_link *uc_l,
                         struct sk_buff_head *xmitq)
{
        struct tipc_link *snd_l = tipc_bc_sndlink(net);

        tipc_bcast_lock(net);
        tipc_link_add_bc_peer(snd_l, uc_l, xmitq);
        tipc_bcbase_select_primary(net);
        tipc_bcbase_calc_bc_threshold(net);
        tipc_bcast_unlock(net);
}

/* tipc_bcast_remove_peer - remove a peer node from broadcast link and bearer
 *
 * RCU is locked, node lock is set
 */
void tipc_bcast_remove_peer(struct net *net, struct tipc_link *rcv_l)
{
        struct tipc_link *snd_l = tipc_bc_sndlink(net);
        struct sk_buff_head *inputq = &tipc_bc_base(net)->inputq;
        struct sk_buff_head xmitq;

        __skb_queue_head_init(&xmitq);

        tipc_bcast_lock(net);
        tipc_link_remove_bc_peer(snd_l, rcv_l, &xmitq);
        tipc_bcbase_select_primary(net);
        tipc_bcbase_calc_bc_threshold(net);
        tipc_bcast_unlock(net);

        tipc_bcbase_xmit(net, &xmitq);

        /* Any socket wakeup messages ? */
        if (!skb_queue_empty(inputq))
                tipc_sk_rcv(net, inputq);
}

int tipc_bclink_reset_stats(struct net *net, struct tipc_link *l)
{
        if (!l)
                return -ENOPROTOOPT;

        tipc_bcast_lock(net);
        tipc_link_reset_stats(l);
        tipc_bcast_unlock(net);
        return 0;
}

static int tipc_bc_link_set_queue_limits(struct net *net, u32 max_win)
{
        struct tipc_link *l = tipc_bc_sndlink(net);

        if (!l)
                return -ENOPROTOOPT;
        if (max_win < BCLINK_WIN_MIN)
                max_win = BCLINK_WIN_MIN;
        if (max_win > TIPC_MAX_LINK_WIN)
                return -EINVAL;
        tipc_bcast_lock(net);
        tipc_link_set_queue_limits(l, tipc_link_min_win(l), max_win);
        tipc_bcast_unlock(net);
        return 0;
}

static int tipc_bc_link_set_broadcast_mode(struct net *net, u32 bc_mode)
{
        struct tipc_bc_base *bb = tipc_bc_base(net);

        switch (bc_mode) {
        case BCLINK_MODE_BCAST:
                if (!bb->bcast_support)
                        return -ENOPROTOOPT;

                bb->force_bcast = true;
                bb->force_rcast = false;
                break;
        case BCLINK_MODE_RCAST:
                if (!bb->rcast_support)
                        return -ENOPROTOOPT;

                bb->force_bcast = false;
                bb->force_rcast = true;
                break;
        case BCLINK_MODE_SEL:
                if (!bb->bcast_support || !bb->rcast_support)
                        return -ENOPROTOOPT;

                bb->force_bcast = false;
                bb->force_rcast = false;
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

static int tipc_bc_link_set_broadcast_ratio(struct net *net, u32 bc_ratio)
{
        struct tipc_bc_base *bb = tipc_bc_base(net);

        if (!bb->bcast_support || !bb->rcast_support)
                return -ENOPROTOOPT;

        if (bc_ratio > 100 || bc_ratio <= 0)
                return -EINVAL;

        bb->rc_ratio = bc_ratio;
        tipc_bcast_lock(net);
        tipc_bcbase_calc_bc_threshold(net);
        tipc_bcast_unlock(net);

        return 0;
}

int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[])
{
        int err;
        u32 win;
        u32 bc_mode;
        u32 bc_ratio;
        struct nlattr *props[TIPC_NLA_PROP_MAX + 1];

        if (!attrs[TIPC_NLA_LINK_PROP])
                return -EINVAL;

        err = tipc_nl_parse_link_prop(attrs[TIPC_NLA_LINK_PROP], props);
        if (err)
                return err;

        if (!props[TIPC_NLA_PROP_WIN] &&
            !props[TIPC_NLA_PROP_BROADCAST] &&
            !props[TIPC_NLA_PROP_BROADCAST_RATIO]) {
                return -EOPNOTSUPP;
        }

        if (props[TIPC_NLA_PROP_BROADCAST]) {
                bc_mode = nla_get_u32(props[TIPC_NLA_PROP_BROADCAST]);
                err = tipc_bc_link_set_broadcast_mode(net, bc_mode);
        }

        if (!err && props[TIPC_NLA_PROP_BROADCAST_RATIO]) {
                bc_ratio = nla_get_u32(props[TIPC_NLA_PROP_BROADCAST_RATIO]);
                err = tipc_bc_link_set_broadcast_ratio(net, bc_ratio);
        }

        if (!err && props[TIPC_NLA_PROP_WIN]) {
                win = nla_get_u32(props[TIPC_NLA_PROP_WIN]);
                err = tipc_bc_link_set_queue_limits(net, win);
        }

        return err;
}

int tipc_bcast_init(struct net *net)
{
        struct tipc_net *tn = tipc_net(net);
        struct tipc_bc_base *bb = NULL;
        struct tipc_link *l = NULL;

        bb = kzalloc(sizeof(*bb), GFP_KERNEL);
        if (!bb)
                goto enomem;
        tn->bcbase = bb;
        spin_lock_init(&tipc_net(net)->bclock);

        if (!tipc_link_bc_create(net, 0, 0, NULL,
                                 one_page_mtu,
                                 BCLINK_WIN_DEFAULT,
                                 BCLINK_WIN_DEFAULT,
                                 0,
                                 &bb->inputq,
                                 NULL,
                                 NULL,
                                 &l))
                goto enomem;
        bb->link = l;
        tn->bcl = l;
        bb->rc_ratio = 10;
        bb->rcast_support = true;
        return 0;
enomem:
        kfree(bb);
        kfree(l);
        return -ENOMEM;
}

void tipc_bcast_stop(struct net *net)
{
        struct tipc_net *tn = net_generic(net, tipc_net_id);

        synchronize_net();
        kfree(tn->bcbase);
        kfree(tn->bcl);
}

void tipc_nlist_init(struct tipc_nlist *nl, u32 self)
{
        memset(nl, 0, sizeof(*nl));
        INIT_LIST_HEAD(&nl->list);
        nl->self = self;
}

void tipc_nlist_add(struct tipc_nlist *nl, u32 node)
{
        if (node == nl->self)
                nl->local = true;
        else if (tipc_dest_push(&nl->list, node, 0))
                nl->remote++;
}

void tipc_nlist_del(struct tipc_nlist *nl, u32 node)
{
        if (node == nl->self)
                nl->local = false;
        else if (tipc_dest_del(&nl->list, node, 0))
                nl->remote--;
}

void tipc_nlist_purge(struct tipc_nlist *nl)
{
        tipc_dest_list_purge(&nl->list);
        nl->remote = 0;
        nl->local = false;
}

u32 tipc_bcast_get_mode(struct net *net)
{
        struct tipc_bc_base *bb = tipc_bc_base(net);

        if (bb->force_bcast)
                return BCLINK_MODE_BCAST;

        if (bb->force_rcast)
                return BCLINK_MODE_RCAST;

        if (bb->bcast_support && bb->rcast_support)
                return BCLINK_MODE_SEL;

        return 0;
}

u32 tipc_bcast_get_broadcast_ratio(struct net *net)
{
        struct tipc_bc_base *bb = tipc_bc_base(net);

        return bb->rc_ratio;
}

void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq,
                           struct sk_buff_head *inputq)
{
        struct sk_buff *skb, *_skb, *tmp;
        struct tipc_msg *hdr, *_hdr;
        bool match = false;
        u32 node, port;

        skb = skb_peek(inputq);
        if (!skb)
                return;

        hdr = buf_msg(skb);

        if (likely(!msg_is_syn(hdr) && skb_queue_empty(defq)))
                return;

        node = msg_orignode(hdr);
        if (node == tipc_own_addr(net))
                return;

        port = msg_origport(hdr);

        /* Has the twin SYN message already arrived ? */
        skb_queue_walk(defq, _skb) {
                _hdr = buf_msg(_skb);
                if (msg_orignode(_hdr) != node)
                        continue;
                if (msg_origport(_hdr) != port)
                        continue;
                match = true;
                break;
        }

        if (!match) {
                if (!msg_is_syn(hdr))
                        return;
                __skb_dequeue(inputq);
                __skb_queue_tail(defq, skb);
                return;
        }

        /* Deliver non-SYN message from other link, otherwise queue it */
        if (!msg_is_syn(hdr)) {
                if (msg_is_rcast(hdr) != msg_is_rcast(_hdr))
                        return;
                __skb_dequeue(inputq);
                __skb_queue_tail(defq, skb);
                return;
        }

        /* Queue non-SYN/SYN message from same link */
        if (msg_is_rcast(hdr) == msg_is_rcast(_hdr)) {
                __skb_dequeue(inputq);
                __skb_queue_tail(defq, skb);
                return;
        }

        /* Matching SYN messages => return the one with data, if any */
        __skb_unlink(_skb, defq);
        if (msg_data_sz(hdr)) {
                kfree_skb(_skb);
        } else {
                __skb_dequeue(inputq);
                kfree_skb(skb);
                __skb_queue_tail(inputq, _skb);
        }

        /* Deliver subsequent non-SYN messages from same peer */
        skb_queue_walk_safe(defq, _skb, tmp) {
                _hdr = buf_msg(_skb);
                if (msg_orignode(_hdr) != node)
                        continue;
                if (msg_origport(_hdr) != port)
                        continue;
                if (msg_is_syn(_hdr))
                        break;
                __skb_unlink(_skb, defq);
                __skb_queue_tail(inputq, _skb);
        }
}






























































































































































































































































































































































































































































































































































































 1021 








  566 

















  165 








 2072 











   91 









 1992 










    9 
    8 













    7 






























































































   19 





  199 
   39 
    3 


 1928 




















   57 



















  538 




















    7 














   61 





























































   11 


   13 














  290 










   52 




























 2484 


 2482 














 1470 


 1463 


 1470 


















  341 













  574 



























 2021 










   23 











    8 
  363 
  365 













  106 








   29 































  343 



    7 
















  343 













   56 
   58 


  384 

































  373 
    2 
  371 





















  963 








    1 








 1503 








  750 








 1642 









    4 
 2157 
 1614 












 1419 




























    3 




  174 
    4 
    1 


  193 


















   63 












  295 












   36 












   93 






































  552 












    3 
  276 



  279 










   91 








































   23 














    2 


















































































































  321 









   94 





























   91 












   85 























  567 











   52 

   15 








    2 















































   35 






















    3 















































   98 













    5 








   52 



    3 













    1 






    5 

    7 
    2 
    5 






















    1 



















































   24 













    3 















































































































































    5 














    2 








    4 










































  655 

  328 

















  391 














  700 
  141 












  299 





















    2 








    2 
















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_NETLINK_H
#define __NET_NETLINK_H

#include <linux/types.h>
#include <linux/netlink.h>
#include <linux/jiffies.h>
#include <linux/in6.h>

/* ========================================================================
 *         Netlink Messages and Attributes Interface (As Seen On TV)
 * ------------------------------------------------------------------------
 *                          Messages Interface
 * ------------------------------------------------------------------------
 *
 * Message Format:
 *    <--- nlmsg_total_size(payload)  --->
 *    <-- nlmsg_msg_size(payload) ->
 *   +----------+- - -+-------------+- - -+-------- - -
 *   | nlmsghdr | Pad |   Payload   | Pad | nlmsghdr
 *   +----------+- - -+-------------+- - -+-------- - -
 *   nlmsg_data(nlh)---^                   ^
 *   nlmsg_next(nlh)-----------------------+
 *
 * Payload Format:
 *    <---------------------- nlmsg_len(nlh) --------------------->
 *    <------ hdrlen ------>       <- nlmsg_attrlen(nlh, hdrlen) ->
 *   +----------------------+- - -+--------------------------------+
 *   |     Family Header    | Pad |           Attributes           |
 *   +----------------------+- - -+--------------------------------+
 *   nlmsg_attrdata(nlh, hdrlen)---^
 *
 * Data Structures:
 *   struct nlmsghdr                        netlink message header
 *
 * Message Construction:
 *   nlmsg_new()                        create a new netlink message
 *   nlmsg_put()                        add a netlink message to an skb
 *   nlmsg_put_answer()                        callback based nlmsg_put()
 *   nlmsg_end()                        finalize netlink message
 *   nlmsg_get_pos()                        return current position in message
 *   nlmsg_trim()                        trim part of message
 *   nlmsg_cancel()                        cancel message construction
 *   nlmsg_consume()                        free a netlink message (expected)
 *   nlmsg_free()                        free a netlink message (drop)
 *
 * Message Sending:
 *   nlmsg_multicast()                        multicast message to several groups
 *   nlmsg_unicast()                        unicast a message to a single socket
 *   nlmsg_notify()                        send notification message
 *
 * Message Length Calculations:
 *   nlmsg_msg_size(payload)                length of message w/o padding
 *   nlmsg_total_size(payload)                length of message w/ padding
 *   nlmsg_padlen(payload)                length of padding at tail
 *
 * Message Payload Access:
 *   nlmsg_data(nlh)                        head of message payload
 *   nlmsg_len(nlh)                        length of message payload
 *   nlmsg_attrdata(nlh, hdrlen)        head of attributes data
 *   nlmsg_attrlen(nlh, hdrlen)                length of attributes data
 *
 * Message Parsing:
 *   nlmsg_ok(nlh, remaining)                does nlh fit into remaining bytes?
 *   nlmsg_next(nlh, remaining)                get next netlink message
 *   nlmsg_parse()                        parse attributes of a message
 *   nlmsg_find_attr()                        find an attribute in a message
 *   nlmsg_for_each_msg()                loop over all messages
 *   nlmsg_validate()                        validate netlink message incl. attrs
 *   nlmsg_for_each_attr()                loop over all attributes
 *
 * Misc:
 *   nlmsg_report()                        report back to application?
 *
 * ------------------------------------------------------------------------
 *                          Attributes Interface
 * ------------------------------------------------------------------------
 *
 * Attribute Format:
 *    <------- nla_total_size(payload) ------->
 *    <---- nla_attr_size(payload) ----->
 *   +----------+- - -+- - - - - - - - - +- - -+-------- - -
 *   |  Header  | Pad |     Payload      | Pad |  Header
 *   +----------+- - -+- - - - - - - - - +- - -+-------- - -
 *                     <- nla_len(nla) ->      ^
 *   nla_data(nla)----^                        |
 *   nla_next(nla)-----------------------------'
 *
 * Data Structures:
 *   struct nlattr                        netlink attribute header
 *
 * Attribute Construction:
 *   nla_reserve(skb, type, len)        reserve room for an attribute
 *   nla_reserve_nohdr(skb, len)        reserve room for an attribute w/o hdr
 *   nla_put(skb, type, len, data)        add attribute to skb
 *   nla_put_nohdr(skb, len, data)        add attribute w/o hdr
 *   nla_append(skb, len, data)                append data to skb
 *
 * Attribute Construction for Basic Types:
 *   nla_put_u8(skb, type, value)        add u8 attribute to skb
 *   nla_put_u16(skb, type, value)        add u16 attribute to skb
 *   nla_put_u32(skb, type, value)        add u32 attribute to skb
 *   nla_put_u64_64bit(skb, type,
 *                     value, padattr)        add u64 attribute to skb
 *   nla_put_s8(skb, type, value)        add s8 attribute to skb
 *   nla_put_s16(skb, type, value)        add s16 attribute to skb
 *   nla_put_s32(skb, type, value)        add s32 attribute to skb
 *   nla_put_s64(skb, type, value,
 *               padattr)                add s64 attribute to skb
 *   nla_put_string(skb, type, str)        add string attribute to skb
 *   nla_put_flag(skb, type)                add flag attribute to skb
 *   nla_put_msecs(skb, type, jiffies,
 *                 padattr)                add msecs attribute to skb
 *   nla_put_in_addr(skb, type, addr)        add IPv4 address attribute to skb
 *   nla_put_in6_addr(skb, type, addr)        add IPv6 address attribute to skb
 *
 * Nested Attributes Construction:
 *   nla_nest_start(skb, type)                start a nested attribute
 *   nla_nest_end(skb, nla)                finalize a nested attribute
 *   nla_nest_cancel(skb, nla)                cancel nested attribute construction
 *
 * Attribute Length Calculations:
 *   nla_attr_size(payload)                length of attribute w/o padding
 *   nla_total_size(payload)                length of attribute w/ padding
 *   nla_padlen(payload)                length of padding
 *
 * Attribute Payload Access:
 *   nla_data(nla)                        head of attribute payload
 *   nla_len(nla)                        length of attribute payload
 *
 * Attribute Payload Access for Basic Types:
 *   nla_get_uint(nla)                        get payload for a uint attribute
 *   nla_get_sint(nla)                        get payload for a sint attribute
 *   nla_get_u8(nla)                        get payload for a u8 attribute
 *   nla_get_u16(nla)                        get payload for a u16 attribute
 *   nla_get_u32(nla)                        get payload for a u32 attribute
 *   nla_get_u64(nla)                        get payload for a u64 attribute
 *   nla_get_s8(nla)                        get payload for a s8 attribute
 *   nla_get_s16(nla)                        get payload for a s16 attribute
 *   nla_get_s32(nla)                        get payload for a s32 attribute
 *   nla_get_s64(nla)                        get payload for a s64 attribute
 *   nla_get_flag(nla)                        return 1 if flag is true
 *   nla_get_msecs(nla)                        get payload for a msecs attribute
 *
 *   The same functions also exist with _default().
 *
 * Attribute Misc:
 *   nla_memcpy(dest, nla, count)        copy attribute into memory
 *   nla_memcmp(nla, data, size)        compare attribute with memory area
 *   nla_strscpy(dst, nla, size)        copy attribute to a sized string
 *   nla_strcmp(nla, str)                compare attribute with string
 *
 * Attribute Parsing:
 *   nla_ok(nla, remaining)                does nla fit into remaining bytes?
 *   nla_next(nla, remaining)                get next netlink attribute
 *   nla_validate()                        validate a stream of attributes
 *   nla_validate_nested()                validate a stream of nested attributes
 *   nla_find()                                find attribute in stream of attributes
 *   nla_find_nested()                        find attribute in nested attributes
 *   nla_parse()                        parse and validate stream of attrs
 *   nla_parse_nested()                        parse nested attributes
 *   nla_for_each_attr()                loop over all attributes
 *   nla_for_each_attr_type()                loop over all attributes with the
 *                                        given type
 *   nla_for_each_nested()                loop over the nested attributes
 *   nla_for_each_nested_type()                loop over the nested attributes with
 *                                        the given type
 *=========================================================================
 */

 /**
  * Standard attribute types to specify validation policy
  */
enum {
        NLA_UNSPEC,
        NLA_U8,
        NLA_U16,
        NLA_U32,
        NLA_U64,
        NLA_STRING,
        NLA_FLAG,
        NLA_MSECS,
        NLA_NESTED,
        NLA_NESTED_ARRAY,
        NLA_NUL_STRING,
        NLA_BINARY,
        NLA_S8,
        NLA_S16,
        NLA_S32,
        NLA_S64,
        NLA_BITFIELD32,
        NLA_REJECT,
        NLA_BE16,
        NLA_BE32,
        NLA_SINT,
        NLA_UINT,
        __NLA_TYPE_MAX,
};

#define NLA_TYPE_MAX (__NLA_TYPE_MAX - 1)

struct netlink_range_validation {
        u64 min, max;
};

struct netlink_range_validation_signed {
        s64 min, max;
};

enum nla_policy_validation {
        NLA_VALIDATE_NONE,
        NLA_VALIDATE_RANGE,
        NLA_VALIDATE_RANGE_WARN_TOO_LONG,
        NLA_VALIDATE_MIN,
        NLA_VALIDATE_MAX,
        NLA_VALIDATE_MASK,
        NLA_VALIDATE_RANGE_PTR,
        NLA_VALIDATE_FUNCTION,
};

/**
 * struct nla_policy - attribute validation policy
 * @type: Type of attribute or NLA_UNSPEC
 * @validation_type: type of attribute validation done in addition to
 *        type-specific validation (e.g. range, function call), see
 *        &enum nla_policy_validation
 * @len: Type specific length of payload
 *
 * Policies are defined as arrays of this struct, the array must be
 * accessible by attribute type up to the highest identifier to be expected.
 *
 * Meaning of `len' field:
 *    NLA_STRING           Maximum length of string
 *    NLA_NUL_STRING       Maximum length of string (excluding NUL)
 *    NLA_FLAG             Unused
 *    NLA_BINARY           Maximum length of attribute payload
 *                         (but see also below with the validation type)
 *    NLA_NESTED,
 *    NLA_NESTED_ARRAY     Length verification is done by checking len of
 *                         nested header (or empty); len field is used if
 *                         nested_policy is also used, for the max attr
 *                         number in the nested policy.
 *    NLA_SINT, NLA_UINT,
 *    NLA_U8, NLA_U16,
 *    NLA_U32, NLA_U64,
 *    NLA_S8, NLA_S16,
 *    NLA_S32, NLA_S64,
 *    NLA_BE16, NLA_BE32,
 *    NLA_MSECS            Leaving the length field zero will verify the
 *                         given type fits, using it verifies minimum length
 *                         just like "All other"
 *    NLA_BITFIELD32       Unused
 *    NLA_REJECT           Unused
 *    All other            Minimum length of attribute payload
 *
 * Meaning of validation union:
 *    NLA_BITFIELD32       This is a 32-bit bitmap/bitselector attribute and
 *                         `bitfield32_valid' is the u32 value of valid flags
 *    NLA_REJECT           This attribute is always rejected and `reject_message'
 *                         may point to a string to report as the error instead
 *                         of the generic one in extended ACK.
 *    NLA_NESTED           `nested_policy' to a nested policy to validate, must
 *                         also set `len' to the max attribute number. Use the
 *                         provided NLA_POLICY_NESTED() macro.
 *                         Note that nla_parse() will validate, but of course not
 *                         parse, the nested sub-policies.
 *    NLA_NESTED_ARRAY     `nested_policy' points to a nested policy to validate,
 *                         must also set `len' to the max attribute number. Use
 *                         the provided NLA_POLICY_NESTED_ARRAY() macro.
 *                         The difference to NLA_NESTED is the structure:
 *                         NLA_NESTED has the nested attributes directly inside
 *                         while an array has the nested attributes at another
 *                         level down and the attribute types directly in the
 *                         nesting don't matter.
 *    NLA_UINT,
 *    NLA_U8,
 *    NLA_U16,
 *    NLA_U32,
 *    NLA_U64,
 *    NLA_BE16,
 *    NLA_BE32,
 *    NLA_SINT,
 *    NLA_S8,
 *    NLA_S16,
 *    NLA_S32,
 *    NLA_S64              The `min' and `max' fields are used depending on the
 *                         validation_type field, if that is min/max/range then
 *                         the min, max or both are used (respectively) to check
 *                         the value of the integer attribute.
 *                         Note that in the interest of code simplicity and
 *                         struct size both limits are s16, so you cannot
 *                         enforce a range that doesn't fall within the range
 *                         of s16 - do that using the NLA_POLICY_FULL_RANGE()
 *                         or NLA_POLICY_FULL_RANGE_SIGNED() macros instead.
 *                         Use the NLA_POLICY_MIN(), NLA_POLICY_MAX() and
 *                         NLA_POLICY_RANGE() macros.
 *    NLA_UINT,
 *    NLA_U8,
 *    NLA_U16,
 *    NLA_U32,
 *    NLA_U64              If the validation_type field instead is set to
 *                         NLA_VALIDATE_RANGE_PTR, `range' must be a pointer
 *                         to a struct netlink_range_validation that indicates
 *                         the min/max values.
 *                         Use NLA_POLICY_FULL_RANGE().
 *    NLA_SINT,
 *    NLA_S8,
 *    NLA_S16,
 *    NLA_S32,
 *    NLA_S64              If the validation_type field instead is set to
 *                         NLA_VALIDATE_RANGE_PTR, `range_signed' must be a
 *                         pointer to a struct netlink_range_validation_signed
 *                         that indicates the min/max values.
 *                         Use NLA_POLICY_FULL_RANGE_SIGNED().
 *
 *    NLA_BINARY           If the validation type is like the ones for integers
 *                         above, then the min/max length (not value like for
 *                         integers) of the attribute is enforced.
 *
 *    All other            Unused - but note that it's a union
 *
 * Meaning of `validate' field, use via NLA_POLICY_VALIDATE_FN:
 *    NLA_BINARY           Validation function called for the attribute.
 *    All other            Unused - but note that it's a union
 *
 * Example:
 *
 * static const u32 myvalidflags = 0xff231023;
 *
 * static const struct nla_policy my_policy[ATTR_MAX+1] = {
 *         [ATTR_FOO] = { .type = NLA_U16 },
 *        [ATTR_BAR] = { .type = NLA_STRING, .len = BARSIZ },
 *        [ATTR_BAZ] = NLA_POLICY_EXACT_LEN(sizeof(struct mystruct)),
 *        [ATTR_GOO] = NLA_POLICY_BITFIELD32(myvalidflags),
 * };
 */
struct nla_policy {
        u8                type;
        u8                validation_type;
        u16                len;
        union {
                /**
                 * @strict_start_type: first attribute to validate strictly
                 *
                 * This entry is special, and used for the attribute at index 0
                 * only, and specifies special data about the policy, namely it
                 * specifies the "boundary type" where strict length validation
                 * starts for any attribute types >= this value, also, strict
                 * nesting validation starts here.
                 *
                 * Additionally, it means that NLA_UNSPEC is actually NLA_REJECT
                 * for any types >= this, so need to use NLA_POLICY_MIN_LEN() to
                 * get the previous pure { .len = xyz } behaviour. The advantage
                 * of this is that types not specified in the policy will be
                 * rejected.
                 *
                 * For completely new families it should be set to 1 so that the
                 * validation is enforced for all attributes. For existing ones
                 * it should be set at least when new attributes are added to
                 * the enum used by the policy, and be set to the new value that
                 * was added to enforce strict validation from thereon.
                 */
                u16 strict_start_type;

                /* private: use NLA_POLICY_*() to set */
                const u32 bitfield32_valid;
                const u32 mask;
                const char *reject_message;
                const struct nla_policy *nested_policy;
                const struct netlink_range_validation *range;
                const struct netlink_range_validation_signed *range_signed;
                struct {
                        s16 min, max;
                };
                int (*validate)(const struct nlattr *attr,
                                struct netlink_ext_ack *extack);
        };
};

#define NLA_POLICY_ETH_ADDR                NLA_POLICY_EXACT_LEN(ETH_ALEN)
#define NLA_POLICY_ETH_ADDR_COMPAT        NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN)

#define _NLA_POLICY_NESTED(maxattr, policy) \
        { .type = NLA_NESTED, .nested_policy = policy, .len = maxattr }
#define _NLA_POLICY_NESTED_ARRAY(maxattr, policy) \
        { .type = NLA_NESTED_ARRAY, .nested_policy = policy, .len = maxattr }
#define NLA_POLICY_NESTED(policy) \
        _NLA_POLICY_NESTED(ARRAY_SIZE(policy) - 1, policy)
#define NLA_POLICY_NESTED_ARRAY(policy) \
        _NLA_POLICY_NESTED_ARRAY(ARRAY_SIZE(policy) - 1, policy)
#define NLA_POLICY_BITFIELD32(valid) \
        { .type = NLA_BITFIELD32, .bitfield32_valid = valid }

#define __NLA_IS_UINT_TYPE(tp)                                        \
        (tp == NLA_U8 || tp == NLA_U16 || tp == NLA_U32 ||        \
         tp == NLA_U64 || tp == NLA_UINT ||                        \
         tp == NLA_BE16 || tp == NLA_BE32)
#define __NLA_IS_SINT_TYPE(tp)                                                \
        (tp == NLA_S8 || tp == NLA_S16 || tp == NLA_S32 || tp == NLA_S64 || \
         tp == NLA_SINT)

#define __NLA_ENSURE(condition) BUILD_BUG_ON_ZERO(!(condition))
#define NLA_ENSURE_UINT_TYPE(tp)                        \
        (__NLA_ENSURE(__NLA_IS_UINT_TYPE(tp)) + tp)
#define NLA_ENSURE_UINT_OR_BINARY_TYPE(tp)                \
        (__NLA_ENSURE(__NLA_IS_UINT_TYPE(tp) ||        \
                      tp == NLA_MSECS ||                \
                      tp == NLA_BINARY) + tp)
#define NLA_ENSURE_SINT_TYPE(tp)                        \
        (__NLA_ENSURE(__NLA_IS_SINT_TYPE(tp)) + tp)
#define NLA_ENSURE_INT_OR_BINARY_TYPE(tp)                \
        (__NLA_ENSURE(__NLA_IS_UINT_TYPE(tp) ||                \
                      __NLA_IS_SINT_TYPE(tp) ||                \
                      tp == NLA_MSECS ||                \
                      tp == NLA_BINARY) + tp)
#define NLA_ENSURE_NO_VALIDATION_PTR(tp)                \
        (__NLA_ENSURE(tp != NLA_BITFIELD32 &&                \
                      tp != NLA_REJECT &&                \
                      tp != NLA_NESTED &&                \
                      tp != NLA_NESTED_ARRAY) + tp)

#define NLA_POLICY_RANGE(tp, _min, _max) {                \
        .type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp),        \
        .validation_type = NLA_VALIDATE_RANGE,                \
        .min = _min,                                        \
        .max = _max                                        \
}

#define NLA_POLICY_FULL_RANGE(tp, _range) {                \
        .type = NLA_ENSURE_UINT_OR_BINARY_TYPE(tp),        \
        .validation_type = NLA_VALIDATE_RANGE_PTR,        \
        .range = _range,                                \
}

#define NLA_POLICY_FULL_RANGE_SIGNED(tp, _range) {        \
        .type = NLA_ENSURE_SINT_TYPE(tp),                \
        .validation_type = NLA_VALIDATE_RANGE_PTR,        \
        .range_signed = _range,                                \
}

#define NLA_POLICY_MIN(tp, _min) {                        \
        .type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp),        \
        .validation_type = NLA_VALIDATE_MIN,                \
        .min = _min,                                        \
}

#define NLA_POLICY_MAX(tp, _max) {                        \
        .type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp),        \
        .validation_type = NLA_VALIDATE_MAX,                \
        .max = _max,                                        \
}

#define NLA_POLICY_MASK(tp, _mask) {                        \
        .type = NLA_ENSURE_UINT_TYPE(tp),                \
        .validation_type = NLA_VALIDATE_MASK,                \
        .mask = _mask,                                        \
}

#define NLA_POLICY_VALIDATE_FN(tp, fn, ...) {                \
        .type = NLA_ENSURE_NO_VALIDATION_PTR(tp),        \
        .validation_type = NLA_VALIDATE_FUNCTION,        \
        .validate = fn,                                        \
        .len = __VA_ARGS__ + 0,                                \
}

#define NLA_POLICY_EXACT_LEN(_len)        NLA_POLICY_RANGE(NLA_BINARY, _len, _len)
#define NLA_POLICY_EXACT_LEN_WARN(_len) {                        \
        .type = NLA_BINARY,                                        \
        .validation_type = NLA_VALIDATE_RANGE_WARN_TOO_LONG,        \
        .min = _len,                                                \
        .max = _len                                                \
}
#define NLA_POLICY_MIN_LEN(_len)        NLA_POLICY_MIN(NLA_BINARY, _len)
#define NLA_POLICY_MAX_LEN(_len)        NLA_POLICY_MAX(NLA_BINARY, _len)

/**
 * struct nl_info - netlink source information
 * @nlh: Netlink message header of original request
 * @nl_net: Network namespace
 * @portid: Netlink PORTID of requesting application
 * @skip_notify: Skip netlink notifications to user space
 * @skip_notify_kernel: Skip selected in-kernel notifications
 */
struct nl_info {
        struct nlmsghdr                *nlh;
        struct net                *nl_net;
        u32                        portid;
        u8                        skip_notify:1,
                                skip_notify_kernel:1;
};

/**
 * enum netlink_validation - netlink message/attribute validation levels
 * @NL_VALIDATE_LIBERAL: Old-style "be liberal" validation, not caring about
 *        extra data at the end of the message, attributes being longer than
 *        they should be, or unknown attributes being present.
 * @NL_VALIDATE_TRAILING: Reject junk data encountered after attribute parsing.
 * @NL_VALIDATE_MAXTYPE: Reject attributes > max type; Together with _TRAILING
 *        this is equivalent to the old nla_parse_strict()/nlmsg_parse_strict().
 * @NL_VALIDATE_UNSPEC: Reject attributes with NLA_UNSPEC in the policy.
 *        This can safely be set by the kernel when the given policy has no
 *        NLA_UNSPEC anymore, and can thus be used to ensure policy entries
 *        are enforced going forward.
 * @NL_VALIDATE_STRICT_ATTRS: strict attribute policy parsing (e.g.
 *        U8, U16, U32 must have exact size, etc.)
 * @NL_VALIDATE_NESTED: Check that NLA_F_NESTED is set for NLA_NESTED(_ARRAY)
 *        and unset for other policies.
 */
enum netlink_validation {
        NL_VALIDATE_LIBERAL = 0,
        NL_VALIDATE_TRAILING = BIT(0),
        NL_VALIDATE_MAXTYPE = BIT(1),
        NL_VALIDATE_UNSPEC = BIT(2),
        NL_VALIDATE_STRICT_ATTRS = BIT(3),
        NL_VALIDATE_NESTED = BIT(4),
};

#define NL_VALIDATE_DEPRECATED_STRICT (NL_VALIDATE_TRAILING |\
                                       NL_VALIDATE_MAXTYPE)
#define NL_VALIDATE_STRICT (NL_VALIDATE_TRAILING |\
                            NL_VALIDATE_MAXTYPE |\
                            NL_VALIDATE_UNSPEC |\
                            NL_VALIDATE_STRICT_ATTRS |\
                            NL_VALIDATE_NESTED)

int netlink_rcv_skb(struct sk_buff *skb,
                    int (*cb)(struct sk_buff *, struct nlmsghdr *,
                              struct netlink_ext_ack *));
int nlmsg_notify(struct sock *sk, struct sk_buff *skb, u32 portid,
                 unsigned int group, int report, gfp_t flags);

int __nla_validate(const struct nlattr *head, int len, int maxtype,
                   const struct nla_policy *policy, unsigned int validate,
                   struct netlink_ext_ack *extack);
int __nla_parse(struct nlattr **tb, int maxtype, const struct nlattr *head,
                int len, const struct nla_policy *policy, unsigned int validate,
                struct netlink_ext_ack *extack);
int nla_policy_len(const struct nla_policy *, int);
struct nlattr *nla_find(const struct nlattr *head, int len, int attrtype);
ssize_t nla_strscpy(char *dst, const struct nlattr *nla, size_t dstsize);
char *nla_strdup(const struct nlattr *nla, gfp_t flags);
int nla_memcpy(void *dest, const struct nlattr *src, int count);
int nla_memcmp(const struct nlattr *nla, const void *data, size_t size);
int nla_strcmp(const struct nlattr *nla, const char *str);
struct nlattr *__nla_reserve(struct sk_buff *skb, int attrtype, int attrlen);
struct nlattr *__nla_reserve_64bit(struct sk_buff *skb, int attrtype,
                                   int attrlen, int padattr);
void *__nla_reserve_nohdr(struct sk_buff *skb, int attrlen);
struct nlattr *nla_reserve(struct sk_buff *skb, int attrtype, int attrlen);
struct nlattr *nla_reserve_64bit(struct sk_buff *skb, int attrtype,
                                 int attrlen, int padattr);
void *nla_reserve_nohdr(struct sk_buff *skb, int attrlen);
void __nla_put(struct sk_buff *skb, int attrtype, int attrlen,
               const void *data);
void __nla_put_64bit(struct sk_buff *skb, int attrtype, int attrlen,
                     const void *data, int padattr);
void __nla_put_nohdr(struct sk_buff *skb, int attrlen, const void *data);
int nla_put(struct sk_buff *skb, int attrtype, int attrlen, const void *data);
int nla_put_64bit(struct sk_buff *skb, int attrtype, int attrlen,
                  const void *data, int padattr);
int nla_put_nohdr(struct sk_buff *skb, int attrlen, const void *data);
int nla_append(struct sk_buff *skb, int attrlen, const void *data);

/**************************************************************************
 * Netlink Messages
 **************************************************************************/

/**
 * nlmsg_msg_size - length of netlink message not including padding
 * @payload: length of message payload
 */
static inline int nlmsg_msg_size(int payload)
{
        return NLMSG_HDRLEN + payload;
}

/**
 * nlmsg_total_size - length of netlink message including padding
 * @payload: length of message payload
 */
static inline int nlmsg_total_size(int payload)
{
        return NLMSG_ALIGN(nlmsg_msg_size(payload));
}

/**
 * nlmsg_padlen - length of padding at the message's tail
 * @payload: length of message payload
 */
static inline int nlmsg_padlen(int payload)
{
        return nlmsg_total_size(payload) - nlmsg_msg_size(payload);
}

/**
 * nlmsg_data - head of message payload
 * @nlh: netlink message header
 */
static inline void *nlmsg_data(const struct nlmsghdr *nlh)
{
        return (unsigned char *) nlh + NLMSG_HDRLEN;
}

/**
 * nlmsg_len - length of message payload
 * @nlh: netlink message header
 */
static inline int nlmsg_len(const struct nlmsghdr *nlh)
{
        return nlh->nlmsg_len - NLMSG_HDRLEN;
}

/**
 * nlmsg_attrdata - head of attributes data
 * @nlh: netlink message header
 * @hdrlen: length of family specific header
 */
static inline struct nlattr *nlmsg_attrdata(const struct nlmsghdr *nlh,
                                            int hdrlen)
{
        unsigned char *data = nlmsg_data(nlh);
        return (struct nlattr *) (data + NLMSG_ALIGN(hdrlen));
}

/**
 * nlmsg_attrlen - length of attributes data
 * @nlh: netlink message header
 * @hdrlen: length of family specific header
 */
static inline int nlmsg_attrlen(const struct nlmsghdr *nlh, int hdrlen)
{
        return nlmsg_len(nlh) - NLMSG_ALIGN(hdrlen);
}

/**
 * nlmsg_ok - check if the netlink message fits into the remaining bytes
 * @nlh: netlink message header
 * @remaining: number of bytes remaining in message stream
 */
static inline int nlmsg_ok(const struct nlmsghdr *nlh, int remaining)
{
        return (remaining >= (int) sizeof(struct nlmsghdr) &&
                nlh->nlmsg_len >= sizeof(struct nlmsghdr) &&
                nlh->nlmsg_len <= remaining);
}

/**
 * nlmsg_next - next netlink message in message stream
 * @nlh: netlink message header
 * @remaining: number of bytes remaining in message stream
 *
 * Returns the next netlink message in the message stream and
 * decrements remaining by the size of the current message.
 */
static inline struct nlmsghdr *
nlmsg_next(const struct nlmsghdr *nlh, int *remaining)
{
        int totlen = NLMSG_ALIGN(nlh->nlmsg_len);

        *remaining -= totlen;

        return (struct nlmsghdr *) ((unsigned char *) nlh + totlen);
}

/**
 * nla_parse - Parse a stream of attributes into a tb buffer
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @head: head of attribute stream
 * @len: length of attribute stream
 * @policy: validation policy
 * @extack: extended ACK pointer
 *
 * Parses a stream of attributes and stores a pointer to each attribute in
 * the tb array accessible via the attribute type. Attributes with a type
 * exceeding maxtype will be rejected, policy must be specified, attributes
 * will be validated in the strictest way possible.
 *
 * Returns 0 on success or a negative error code.
 */
static inline int nla_parse(struct nlattr **tb, int maxtype,
                            const struct nlattr *head, int len,
                            const struct nla_policy *policy,
                            struct netlink_ext_ack *extack)
{
        return __nla_parse(tb, maxtype, head, len, policy,
                           NL_VALIDATE_STRICT, extack);
}

/**
 * nla_parse_deprecated - Parse a stream of attributes into a tb buffer
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @head: head of attribute stream
 * @len: length of attribute stream
 * @policy: validation policy
 * @extack: extended ACK pointer
 *
 * Parses a stream of attributes and stores a pointer to each attribute in
 * the tb array accessible via the attribute type. Attributes with a type
 * exceeding maxtype will be ignored and attributes from the policy are not
 * always strictly validated (only for new attributes).
 *
 * Returns 0 on success or a negative error code.
 */
static inline int nla_parse_deprecated(struct nlattr **tb, int maxtype,
                                       const struct nlattr *head, int len,
                                       const struct nla_policy *policy,
                                       struct netlink_ext_ack *extack)
{
        return __nla_parse(tb, maxtype, head, len, policy,
                           NL_VALIDATE_LIBERAL, extack);
}

/**
 * nla_parse_deprecated_strict - Parse a stream of attributes into a tb buffer
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @head: head of attribute stream
 * @len: length of attribute stream
 * @policy: validation policy
 * @extack: extended ACK pointer
 *
 * Parses a stream of attributes and stores a pointer to each attribute in
 * the tb array accessible via the attribute type. Attributes with a type
 * exceeding maxtype will be rejected as well as trailing data, but the
 * policy is not completely strictly validated (only for new attributes).
 *
 * Returns 0 on success or a negative error code.
 */
static inline int nla_parse_deprecated_strict(struct nlattr **tb, int maxtype,
                                              const struct nlattr *head,
                                              int len,
                                              const struct nla_policy *policy,
                                              struct netlink_ext_ack *extack)
{
        return __nla_parse(tb, maxtype, head, len, policy,
                           NL_VALIDATE_DEPRECATED_STRICT, extack);
}

/**
 * __nlmsg_parse - parse attributes of a netlink message
 * @nlh: netlink message header
 * @hdrlen: length of family specific header
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @policy: validation policy
 * @validate: validation strictness
 * @extack: extended ACK report struct
 *
 * See nla_parse()
 */
static inline int __nlmsg_parse(const struct nlmsghdr *nlh, int hdrlen,
                                struct nlattr *tb[], int maxtype,
                                const struct nla_policy *policy,
                                unsigned int validate,
                                struct netlink_ext_ack *extack)
{
        if (nlh->nlmsg_len < nlmsg_msg_size(hdrlen)) {
                NL_SET_ERR_MSG(extack, "Invalid header length");
                return -EINVAL;
        }

        return __nla_parse(tb, maxtype, nlmsg_attrdata(nlh, hdrlen),
                           nlmsg_attrlen(nlh, hdrlen), policy, validate,
                           extack);
}

/**
 * nlmsg_parse - parse attributes of a netlink message
 * @nlh: netlink message header
 * @hdrlen: length of family specific header
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @policy: validation policy
 * @extack: extended ACK report struct
 *
 * See nla_parse()
 */
static inline int nlmsg_parse(const struct nlmsghdr *nlh, int hdrlen,
                              struct nlattr *tb[], int maxtype,
                              const struct nla_policy *policy,
                              struct netlink_ext_ack *extack)
{
        return __nlmsg_parse(nlh, hdrlen, tb, maxtype, policy,
                             NL_VALIDATE_STRICT, extack);
}

/**
 * nlmsg_parse_deprecated - parse attributes of a netlink message
 * @nlh: netlink message header
 * @hdrlen: length of family specific header
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @policy: validation policy
 * @extack: extended ACK report struct
 *
 * See nla_parse_deprecated()
 */
static inline int nlmsg_parse_deprecated(const struct nlmsghdr *nlh, int hdrlen,
                                         struct nlattr *tb[], int maxtype,
                                         const struct nla_policy *policy,
                                         struct netlink_ext_ack *extack)
{
        return __nlmsg_parse(nlh, hdrlen, tb, maxtype, policy,
                             NL_VALIDATE_LIBERAL, extack);
}

/**
 * nlmsg_parse_deprecated_strict - parse attributes of a netlink message
 * @nlh: netlink message header
 * @hdrlen: length of family specific header
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @policy: validation policy
 * @extack: extended ACK report struct
 *
 * See nla_parse_deprecated_strict()
 */
static inline int
nlmsg_parse_deprecated_strict(const struct nlmsghdr *nlh, int hdrlen,
                              struct nlattr *tb[], int maxtype,
                              const struct nla_policy *policy,
                              struct netlink_ext_ack *extack)
{
        return __nlmsg_parse(nlh, hdrlen, tb, maxtype, policy,
                             NL_VALIDATE_DEPRECATED_STRICT, extack);
}

/**
 * nlmsg_find_attr - find a specific attribute in a netlink message
 * @nlh: netlink message header
 * @hdrlen: length of family specific header
 * @attrtype: type of attribute to look for
 *
 * Returns the first attribute which matches the specified type.
 */
static inline struct nlattr *nlmsg_find_attr(const struct nlmsghdr *nlh,
                                             int hdrlen, int attrtype)
{
        return nla_find(nlmsg_attrdata(nlh, hdrlen),
                        nlmsg_attrlen(nlh, hdrlen), attrtype);
}

/**
 * nla_validate_deprecated - Validate a stream of attributes
 * @head: head of attribute stream
 * @len: length of attribute stream
 * @maxtype: maximum attribute type to be expected
 * @policy: validation policy
 * @extack: extended ACK report struct
 *
 * Validates all attributes in the specified attribute stream against the
 * specified policy. Validation is done in liberal mode.
 * See documentation of struct nla_policy for more details.
 *
 * Returns 0 on success or a negative error code.
 */
static inline int nla_validate_deprecated(const struct nlattr *head, int len,
                                          int maxtype,
                                          const struct nla_policy *policy,
                                          struct netlink_ext_ack *extack)
{
        return __nla_validate(head, len, maxtype, policy, NL_VALIDATE_LIBERAL,
                              extack);
}

/**
 * nla_validate - Validate a stream of attributes
 * @head: head of attribute stream
 * @len: length of attribute stream
 * @maxtype: maximum attribute type to be expected
 * @policy: validation policy
 * @extack: extended ACK report struct
 *
 * Validates all attributes in the specified attribute stream against the
 * specified policy. Validation is done in strict mode.
 * See documentation of struct nla_policy for more details.
 *
 * Returns 0 on success or a negative error code.
 */
static inline int nla_validate(const struct nlattr *head, int len, int maxtype,
                               const struct nla_policy *policy,
                               struct netlink_ext_ack *extack)
{
        return __nla_validate(head, len, maxtype, policy, NL_VALIDATE_STRICT,
                              extack);
}

/**
 * nlmsg_validate_deprecated - validate a netlink message including attributes
 * @nlh: netlinket message header
 * @hdrlen: length of family specific header
 * @maxtype: maximum attribute type to be expected
 * @policy: validation policy
 * @extack: extended ACK report struct
 */
static inline int nlmsg_validate_deprecated(const struct nlmsghdr *nlh,
                                            int hdrlen, int maxtype,
                                            const struct nla_policy *policy,
                                            struct netlink_ext_ack *extack)
{
        if (nlh->nlmsg_len < nlmsg_msg_size(hdrlen))
                return -EINVAL;

        return __nla_validate(nlmsg_attrdata(nlh, hdrlen),
                              nlmsg_attrlen(nlh, hdrlen), maxtype,
                              policy, NL_VALIDATE_LIBERAL, extack);
}



/**
 * nlmsg_report - need to report back to application?
 * @nlh: netlink message header
 *
 * Returns 1 if a report back to the application is requested.
 */
static inline int nlmsg_report(const struct nlmsghdr *nlh)
{
        return nlh ? !!(nlh->nlmsg_flags & NLM_F_ECHO) : 0;
}

/**
 * nlmsg_seq - return the seq number of netlink message
 * @nlh: netlink message header
 *
 * Returns 0 if netlink message is NULL
 */
static inline u32 nlmsg_seq(const struct nlmsghdr *nlh)
{
        return nlh ? nlh->nlmsg_seq : 0;
}

/**
 * nlmsg_for_each_attr - iterate over a stream of attributes
 * @pos: loop counter, set to current attribute
 * @nlh: netlink message header
 * @hdrlen: length of family specific header
 * @rem: initialized to len, holds bytes currently remaining in stream
 */
#define nlmsg_for_each_attr(pos, nlh, hdrlen, rem) \
        nla_for_each_attr(pos, nlmsg_attrdata(nlh, hdrlen), \
                          nlmsg_attrlen(nlh, hdrlen), rem)

/**
 * nlmsg_put - Add a new netlink message to an skb
 * @skb: socket buffer to store message in
 * @portid: netlink PORTID of requesting application
 * @seq: sequence number of message
 * @type: message type
 * @payload: length of message payload
 * @flags: message flags
 *
 * Returns NULL if the tailroom of the skb is insufficient to store
 * the message header and payload.
 */
static inline struct nlmsghdr *nlmsg_put(struct sk_buff *skb, u32 portid, u32 seq,
                                         int type, int payload, int flags)
{
        if (unlikely(skb_tailroom(skb) < nlmsg_total_size(payload)))
                return NULL;

        return __nlmsg_put(skb, portid, seq, type, payload, flags);
}

/**
 * nlmsg_append - Add more data to a nlmsg in a skb
 * @skb: socket buffer to store message in
 * @size: length of message payload
 *
 * Append data to an existing nlmsg, used when constructing a message
 * with multiple fixed-format headers (which is rare).
 * Returns NULL if the tailroom of the skb is insufficient to store
 * the extra payload.
 */
static inline void *nlmsg_append(struct sk_buff *skb, u32 size)
{
        if (unlikely(skb_tailroom(skb) < NLMSG_ALIGN(size)))
                return NULL;

        if (NLMSG_ALIGN(size) - size)
                memset(skb_tail_pointer(skb) + size, 0,
                       NLMSG_ALIGN(size) - size);
        return __skb_put(skb, NLMSG_ALIGN(size));
}

/**
 * nlmsg_put_answer - Add a new callback based netlink message to an skb
 * @skb: socket buffer to store message in
 * @cb: netlink callback
 * @type: message type
 * @payload: length of message payload
 * @flags: message flags
 *
 * Returns NULL if the tailroom of the skb is insufficient to store
 * the message header and payload.
 */
static inline struct nlmsghdr *nlmsg_put_answer(struct sk_buff *skb,
                                                struct netlink_callback *cb,
                                                int type, int payload,
                                                int flags)
{
        return nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
                         type, payload, flags);
}

/**
 * nlmsg_new - Allocate a new netlink message
 * @payload: size of the message payload
 * @flags: the type of memory to allocate.
 *
 * Use NLMSG_DEFAULT_SIZE if the size of the payload isn't known
 * and a good default is needed.
 */
static inline struct sk_buff *nlmsg_new(size_t payload, gfp_t flags)
{
        return alloc_skb(nlmsg_total_size(payload), flags);
}

/**
 * nlmsg_new_large - Allocate a new netlink message with non-contiguous
 * physical memory
 * @payload: size of the message payload
 *
 * The allocated skb is unable to have frag page for shinfo->frags*,
 * as the NULL setting for skb->head in netlink_skb_destructor() will
 * bypass most of the handling in skb_release_data()
 */
static inline struct sk_buff *nlmsg_new_large(size_t payload)
{
        return netlink_alloc_large_skb(nlmsg_total_size(payload), 0);
}

/**
 * nlmsg_end - Finalize a netlink message
 * @skb: socket buffer the message is stored in
 * @nlh: netlink message header
 *
 * Corrects the netlink message header to include the appended
 * attributes. Only necessary if attributes have been added to
 * the message.
 */
static inline void nlmsg_end(struct sk_buff *skb, struct nlmsghdr *nlh)
{
        nlh->nlmsg_len = skb_tail_pointer(skb) - (unsigned char *)nlh;
}

/**
 * nlmsg_get_pos - return current position in netlink message
 * @skb: socket buffer the message is stored in
 *
 * Returns a pointer to the current tail of the message.
 */
static inline void *nlmsg_get_pos(struct sk_buff *skb)
{
        return skb_tail_pointer(skb);
}

/**
 * nlmsg_trim - Trim message to a mark
 * @skb: socket buffer the message is stored in
 * @mark: mark to trim to
 *
 * Trims the message to the provided mark.
 */
static inline void nlmsg_trim(struct sk_buff *skb, const void *mark)
{
        if (mark) {
                WARN_ON((unsigned char *) mark < skb->data);
                skb_trim(skb, (unsigned char *) mark - skb->data);
        }
}

/**
 * nlmsg_cancel - Cancel construction of a netlink message
 * @skb: socket buffer the message is stored in
 * @nlh: netlink message header
 *
 * Removes the complete netlink message including all
 * attributes from the socket buffer again.
 */
static inline void nlmsg_cancel(struct sk_buff *skb, struct nlmsghdr *nlh)
{
        nlmsg_trim(skb, nlh);
}

/**
 * nlmsg_free - drop a netlink message
 * @skb: socket buffer of netlink message
 */
static inline void nlmsg_free(struct sk_buff *skb)
{
        kfree_skb(skb);
}

/**
 * nlmsg_consume - free a netlink message
 * @skb: socket buffer of netlink message
 */
static inline void nlmsg_consume(struct sk_buff *skb)
{
        consume_skb(skb);
}

/**
 * nlmsg_multicast_filtered - multicast a netlink message with filter function
 * @sk: netlink socket to spread messages to
 * @skb: netlink message as socket buffer
 * @portid: own netlink portid to avoid sending to yourself
 * @group: multicast group id
 * @flags: allocation flags
 * @filter: filter function
 * @filter_data: filter function private data
 *
 * Return: 0 on success, negative error code for failure.
 */
static inline int nlmsg_multicast_filtered(struct sock *sk, struct sk_buff *skb,
                                           u32 portid, unsigned int group,
                                           gfp_t flags,
                                           netlink_filter_fn filter,
                                           void *filter_data)
{
        int err;

        NETLINK_CB(skb).dst_group = group;

        err = netlink_broadcast_filtered(sk, skb, portid, group, flags,
                                         filter, filter_data);
        if (err > 0)
                err = 0;

        return err;
}

/**
 * nlmsg_multicast - multicast a netlink message
 * @sk: netlink socket to spread messages to
 * @skb: netlink message as socket buffer
 * @portid: own netlink portid to avoid sending to yourself
 * @group: multicast group id
 * @flags: allocation flags
 */
static inline int nlmsg_multicast(struct sock *sk, struct sk_buff *skb,
                                  u32 portid, unsigned int group, gfp_t flags)
{
        return nlmsg_multicast_filtered(sk, skb, portid, group, flags,
                                        NULL, NULL);
}

/**
 * nlmsg_unicast - unicast a netlink message
 * @sk: netlink socket to spread message to
 * @skb: netlink message as socket buffer
 * @portid: netlink portid of the destination socket
 */
static inline int nlmsg_unicast(struct sock *sk, struct sk_buff *skb, u32 portid)
{
        int err;

        err = netlink_unicast(sk, skb, portid, MSG_DONTWAIT);
        if (err > 0)
                err = 0;

        return err;
}

/**
 * nlmsg_for_each_msg - iterate over a stream of messages
 * @pos: loop counter, set to current message
 * @head: head of message stream
 * @len: length of message stream
 * @rem: initialized to len, holds bytes currently remaining in stream
 */
#define nlmsg_for_each_msg(pos, head, len, rem) \
        for (pos = head, rem = len; \
             nlmsg_ok(pos, rem); \
             pos = nlmsg_next(pos, &(rem)))

/**
 * nl_dump_check_consistent - check if sequence is consistent and advertise if not
 * @cb: netlink callback structure that stores the sequence number
 * @nlh: netlink message header to write the flag to
 *
 * This function checks if the sequence (generation) number changed during dump
 * and if it did, advertises it in the netlink message header.
 *
 * The correct way to use it is to set cb->seq to the generation counter when
 * all locks for dumping have been acquired, and then call this function for
 * each message that is generated.
 *
 * Note that due to initialisation concerns, 0 is an invalid sequence number
 * and must not be used by code that uses this functionality.
 */
static inline void
nl_dump_check_consistent(struct netlink_callback *cb,
                         struct nlmsghdr *nlh)
{
        if (cb->prev_seq && cb->seq != cb->prev_seq)
                nlh->nlmsg_flags |= NLM_F_DUMP_INTR;
        cb->prev_seq = cb->seq;
}

/**************************************************************************
 * Netlink Attributes
 **************************************************************************/

/**
 * nla_attr_size - length of attribute not including padding
 * @payload: length of payload
 */
static inline int nla_attr_size(int payload)
{
        return NLA_HDRLEN + payload;
}

/**
 * nla_total_size - total length of attribute including padding
 * @payload: length of payload
 */
static inline int nla_total_size(int payload)
{
        return NLA_ALIGN(nla_attr_size(payload));
}

/**
 * nla_padlen - length of padding at the tail of attribute
 * @payload: length of payload
 */
static inline int nla_padlen(int payload)
{
        return nla_total_size(payload) - nla_attr_size(payload);
}

/**
 * nla_type - attribute type
 * @nla: netlink attribute
 */
static inline int nla_type(const struct nlattr *nla)
{
        return nla->nla_type & NLA_TYPE_MASK;
}

/**
 * nla_data - head of payload
 * @nla: netlink attribute
 */
static inline void *nla_data(const struct nlattr *nla)
{
        return (char *) nla + NLA_HDRLEN;
}

/**
 * nla_len - length of payload
 * @nla: netlink attribute
 */
static inline u16 nla_len(const struct nlattr *nla)
{
        return nla->nla_len - NLA_HDRLEN;
}

/**
 * nla_ok - check if the netlink attribute fits into the remaining bytes
 * @nla: netlink attribute
 * @remaining: number of bytes remaining in attribute stream
 */
static inline int nla_ok(const struct nlattr *nla, int remaining)
{
        return remaining >= (int) sizeof(*nla) &&
               nla->nla_len >= sizeof(*nla) &&
               nla->nla_len <= remaining;
}

/**
 * nla_next - next netlink attribute in attribute stream
 * @nla: netlink attribute
 * @remaining: number of bytes remaining in attribute stream
 *
 * Returns the next netlink attribute in the attribute stream and
 * decrements remaining by the size of the current attribute.
 */
static inline struct nlattr *nla_next(const struct nlattr *nla, int *remaining)
{
        unsigned int totlen = NLA_ALIGN(nla->nla_len);

        *remaining -= totlen;
        return (struct nlattr *) ((char *) nla + totlen);
}

/**
 * nla_find_nested - find attribute in a set of nested attributes
 * @nla: attribute containing the nested attributes
 * @attrtype: type of attribute to look for
 *
 * Returns the first attribute which matches the specified type.
 */
static inline struct nlattr *
nla_find_nested(const struct nlattr *nla, int attrtype)
{
        return nla_find(nla_data(nla), nla_len(nla), attrtype);
}

/**
 * nla_parse_nested - parse nested attributes
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @nla: attribute containing the nested attributes
 * @policy: validation policy
 * @extack: extended ACK report struct
 *
 * See nla_parse()
 */
static inline int nla_parse_nested(struct nlattr *tb[], int maxtype,
                                   const struct nlattr *nla,
                                   const struct nla_policy *policy,
                                   struct netlink_ext_ack *extack)
{
        if (!(nla->nla_type & NLA_F_NESTED)) {
                NL_SET_ERR_MSG_ATTR(extack, nla, "NLA_F_NESTED is missing");
                return -EINVAL;
        }

        return __nla_parse(tb, maxtype, nla_data(nla), nla_len(nla), policy,
                           NL_VALIDATE_STRICT, extack);
}

/**
 * nla_parse_nested_deprecated - parse nested attributes
 * @tb: destination array with maxtype+1 elements
 * @maxtype: maximum attribute type to be expected
 * @nla: attribute containing the nested attributes
 * @policy: validation policy
 * @extack: extended ACK report struct
 *
 * See nla_parse_deprecated()
 */
static inline int nla_parse_nested_deprecated(struct nlattr *tb[], int maxtype,
                                              const struct nlattr *nla,
                                              const struct nla_policy *policy,
                                              struct netlink_ext_ack *extack)
{
        return __nla_parse(tb, maxtype, nla_data(nla), nla_len(nla), policy,
                           NL_VALIDATE_LIBERAL, extack);
}

/**
 * nla_put_u8 - Add a u8 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_u8(struct sk_buff *skb, int attrtype, u8 value)
{
        /* temporary variables to work around GCC PR81715 with asan-stack=1 */
        u8 tmp = value;

        return nla_put(skb, attrtype, sizeof(u8), &tmp);
}

/**
 * nla_put_u16 - Add a u16 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_u16(struct sk_buff *skb, int attrtype, u16 value)
{
        u16 tmp = value;

        return nla_put(skb, attrtype, sizeof(u16), &tmp);
}

/**
 * nla_put_be16 - Add a __be16 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_be16(struct sk_buff *skb, int attrtype, __be16 value)
{
        __be16 tmp = value;

        return nla_put(skb, attrtype, sizeof(__be16), &tmp);
}

/**
 * nla_put_net16 - Add 16-bit network byte order netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_net16(struct sk_buff *skb, int attrtype, __be16 value)
{
        __be16 tmp = value;

        return nla_put_be16(skb, attrtype | NLA_F_NET_BYTEORDER, tmp);
}

/**
 * nla_put_le16 - Add a __le16 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_le16(struct sk_buff *skb, int attrtype, __le16 value)
{
        __le16 tmp = value;

        return nla_put(skb, attrtype, sizeof(__le16), &tmp);
}

/**
 * nla_put_u32 - Add a u32 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_u32(struct sk_buff *skb, int attrtype, u32 value)
{
        u32 tmp = value;

        return nla_put(skb, attrtype, sizeof(u32), &tmp);
}

/**
 * nla_put_uint - Add a variable-size unsigned int to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_uint(struct sk_buff *skb, int attrtype, u64 value)
{
        u64 tmp64 = value;
        u32 tmp32 = value;

        if (tmp64 == tmp32)
                return nla_put_u32(skb, attrtype, tmp32);
        return nla_put(skb, attrtype, sizeof(u64), &tmp64);
}

/**
 * nla_put_be32 - Add a __be32 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_be32(struct sk_buff *skb, int attrtype, __be32 value)
{
        __be32 tmp = value;

        return nla_put(skb, attrtype, sizeof(__be32), &tmp);
}

/**
 * nla_put_net32 - Add 32-bit network byte order netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_net32(struct sk_buff *skb, int attrtype, __be32 value)
{
        __be32 tmp = value;

        return nla_put_be32(skb, attrtype | NLA_F_NET_BYTEORDER, tmp);
}

/**
 * nla_put_le32 - Add a __le32 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_le32(struct sk_buff *skb, int attrtype, __le32 value)
{
        __le32 tmp = value;

        return nla_put(skb, attrtype, sizeof(__le32), &tmp);
}

/**
 * nla_put_u64_64bit - Add a u64 netlink attribute to a skb and align it
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 * @padattr: attribute type for the padding
 */
static inline int nla_put_u64_64bit(struct sk_buff *skb, int attrtype,
                                    u64 value, int padattr)
{
        u64 tmp = value;

        return nla_put_64bit(skb, attrtype, sizeof(u64), &tmp, padattr);
}

/**
 * nla_put_be64 - Add a __be64 netlink attribute to a socket buffer and align it
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 * @padattr: attribute type for the padding
 */
static inline int nla_put_be64(struct sk_buff *skb, int attrtype, __be64 value,
                               int padattr)
{
        __be64 tmp = value;

        return nla_put_64bit(skb, attrtype, sizeof(__be64), &tmp, padattr);
}

/**
 * nla_put_net64 - Add 64-bit network byte order nlattr to a skb and align it
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 * @padattr: attribute type for the padding
 */
static inline int nla_put_net64(struct sk_buff *skb, int attrtype, __be64 value,
                                int padattr)
{
        __be64 tmp = value;

        return nla_put_be64(skb, attrtype | NLA_F_NET_BYTEORDER, tmp,
                            padattr);
}

/**
 * nla_put_le64 - Add a __le64 netlink attribute to a socket buffer and align it
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 * @padattr: attribute type for the padding
 */
static inline int nla_put_le64(struct sk_buff *skb, int attrtype, __le64 value,
                               int padattr)
{
        __le64 tmp = value;

        return nla_put_64bit(skb, attrtype, sizeof(__le64), &tmp, padattr);
}

/**
 * nla_put_s8 - Add a s8 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_s8(struct sk_buff *skb, int attrtype, s8 value)
{
        s8 tmp = value;

        return nla_put(skb, attrtype, sizeof(s8), &tmp);
}

/**
 * nla_put_s16 - Add a s16 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_s16(struct sk_buff *skb, int attrtype, s16 value)
{
        s16 tmp = value;

        return nla_put(skb, attrtype, sizeof(s16), &tmp);
}

/**
 * nla_put_s32 - Add a s32 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_s32(struct sk_buff *skb, int attrtype, s32 value)
{
        s32 tmp = value;

        return nla_put(skb, attrtype, sizeof(s32), &tmp);
}

/**
 * nla_put_s64 - Add a s64 netlink attribute to a socket buffer and align it
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 * @padattr: attribute type for the padding
 */
static inline int nla_put_s64(struct sk_buff *skb, int attrtype, s64 value,
                              int padattr)
{
        s64 tmp = value;

        return nla_put_64bit(skb, attrtype, sizeof(s64), &tmp, padattr);
}

/**
 * nla_put_sint - Add a variable-size signed int to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: numeric value
 */
static inline int nla_put_sint(struct sk_buff *skb, int attrtype, s64 value)
{
        s64 tmp64 = value;
        s32 tmp32 = value;

        if (tmp64 == tmp32)
                return nla_put_s32(skb, attrtype, tmp32);
        return nla_put(skb, attrtype, sizeof(s64), &tmp64);
}

/**
 * nla_put_string - Add a string netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @str: NUL terminated string
 */
static inline int nla_put_string(struct sk_buff *skb, int attrtype,
                                 const char *str)
{
        return nla_put(skb, attrtype, strlen(str) + 1, str);
}

/**
 * nla_put_flag - Add a flag netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 */
static inline int nla_put_flag(struct sk_buff *skb, int attrtype)
{
        return nla_put(skb, attrtype, 0, NULL);
}

/**
 * nla_put_msecs - Add a msecs netlink attribute to a skb and align it
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @njiffies: number of jiffies to convert to msecs
 * @padattr: attribute type for the padding
 */
static inline int nla_put_msecs(struct sk_buff *skb, int attrtype,
                                unsigned long njiffies, int padattr)
{
        u64 tmp = jiffies_to_msecs(njiffies);

        return nla_put_64bit(skb, attrtype, sizeof(u64), &tmp, padattr);
}

/**
 * nla_put_in_addr - Add an IPv4 address netlink attribute to a socket
 * buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @addr: IPv4 address
 */
static inline int nla_put_in_addr(struct sk_buff *skb, int attrtype,
                                  __be32 addr)
{
        __be32 tmp = addr;

        return nla_put_be32(skb, attrtype, tmp);
}

/**
 * nla_put_in6_addr - Add an IPv6 address netlink attribute to a socket
 * buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @addr: IPv6 address
 */
static inline int nla_put_in6_addr(struct sk_buff *skb, int attrtype,
                                   const struct in6_addr *addr)
{
        return nla_put(skb, attrtype, sizeof(*addr), addr);
}

/**
 * nla_put_bitfield32 - Add a bitfield32 netlink attribute to a socket buffer
 * @skb: socket buffer to add attribute to
 * @attrtype: attribute type
 * @value: value carrying bits
 * @selector: selector of valid bits
 */
static inline int nla_put_bitfield32(struct sk_buff *skb, int attrtype,
                                     __u32 value, __u32 selector)
{
        struct nla_bitfield32 tmp = { value, selector, };

        return nla_put(skb, attrtype, sizeof(tmp), &tmp);
}

/**
 * nla_get_u32 - return payload of u32 attribute
 * @nla: u32 netlink attribute
 */
static inline u32 nla_get_u32(const struct nlattr *nla)
{
        return *(u32 *) nla_data(nla);
}

/**
 * nla_get_u32_default - return payload of u32 attribute or default
 * @nla: u32 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline u32 nla_get_u32_default(const struct nlattr *nla, u32 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_u32(nla);
}

/**
 * nla_get_be32 - return payload of __be32 attribute
 * @nla: __be32 netlink attribute
 */
static inline __be32 nla_get_be32(const struct nlattr *nla)
{
        return *(__be32 *) nla_data(nla);
}

/**
 * nla_get_be32_default - return payload of be32 attribute or default
 * @nla: __be32 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline __be32 nla_get_be32_default(const struct nlattr *nla,
                                          __be32 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_be32(nla);
}

/**
 * nla_get_le32 - return payload of __le32 attribute
 * @nla: __le32 netlink attribute
 */
static inline __le32 nla_get_le32(const struct nlattr *nla)
{
        return *(__le32 *) nla_data(nla);
}

/**
 * nla_get_le32_default - return payload of le32 attribute or default
 * @nla: __le32 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline __le32 nla_get_le32_default(const struct nlattr *nla,
                                          __le32 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_le32(nla);
}

/**
 * nla_get_u16 - return payload of u16 attribute
 * @nla: u16 netlink attribute
 */
static inline u16 nla_get_u16(const struct nlattr *nla)
{
        return *(u16 *) nla_data(nla);
}

/**
 * nla_get_u16_default - return payload of u16 attribute or default
 * @nla: u16 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline u16 nla_get_u16_default(const struct nlattr *nla, u16 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_u16(nla);
}

/**
 * nla_get_be16 - return payload of __be16 attribute
 * @nla: __be16 netlink attribute
 */
static inline __be16 nla_get_be16(const struct nlattr *nla)
{
        return *(__be16 *) nla_data(nla);
}

/**
 * nla_get_be16_default - return payload of be16 attribute or default
 * @nla: __be16 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline __be16 nla_get_be16_default(const struct nlattr *nla,
                                          __be16 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_be16(nla);
}

/**
 * nla_get_le16 - return payload of __le16 attribute
 * @nla: __le16 netlink attribute
 */
static inline __le16 nla_get_le16(const struct nlattr *nla)
{
        return *(__le16 *) nla_data(nla);
}

/**
 * nla_get_le16_default - return payload of le16 attribute or default
 * @nla: __le16 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline __le16 nla_get_le16_default(const struct nlattr *nla,
                                          __le16 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_le16(nla);
}

/**
 * nla_get_u8 - return payload of u8 attribute
 * @nla: u8 netlink attribute
 */
static inline u8 nla_get_u8(const struct nlattr *nla)
{
        return *(u8 *) nla_data(nla);
}

/**
 * nla_get_u8_default - return payload of u8 attribute or default
 * @nla: u8 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline u8 nla_get_u8_default(const struct nlattr *nla, u8 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_u8(nla);
}

/**
 * nla_get_u64 - return payload of u64 attribute
 * @nla: u64 netlink attribute
 */
static inline u64 nla_get_u64(const struct nlattr *nla)
{
        u64 tmp;

        nla_memcpy(&tmp, nla, sizeof(tmp));

        return tmp;
}

/**
 * nla_get_u64_default - return payload of u64 attribute or default
 * @nla: u64 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline u64 nla_get_u64_default(const struct nlattr *nla, u64 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_u64(nla);
}

/**
 * nla_get_uint - return payload of uint attribute
 * @nla: uint netlink attribute
 */
static inline u64 nla_get_uint(const struct nlattr *nla)
{
        if (nla_len(nla) == sizeof(u32))
                return nla_get_u32(nla);
        return nla_get_u64(nla);
}

/**
 * nla_get_uint_default - return payload of uint attribute or default
 * @nla: uint netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline u64 nla_get_uint_default(const struct nlattr *nla, u64 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_uint(nla);
}

/**
 * nla_get_be64 - return payload of __be64 attribute
 * @nla: __be64 netlink attribute
 */
static inline __be64 nla_get_be64(const struct nlattr *nla)
{
        __be64 tmp;

        nla_memcpy(&tmp, nla, sizeof(tmp));

        return tmp;
}

/**
 * nla_get_be64_default - return payload of be64 attribute or default
 * @nla: __be64 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline __be64 nla_get_be64_default(const struct nlattr *nla,
                                          __be64 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_be64(nla);
}

/**
 * nla_get_le64 - return payload of __le64 attribute
 * @nla: __le64 netlink attribute
 */
static inline __le64 nla_get_le64(const struct nlattr *nla)
{
        return *(__le64 *) nla_data(nla);
}

/**
 * nla_get_le64_default - return payload of le64 attribute or default
 * @nla: __le64 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline __le64 nla_get_le64_default(const struct nlattr *nla,
                                          __le64 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_le64(nla);
}

/**
 * nla_get_s32 - return payload of s32 attribute
 * @nla: s32 netlink attribute
 */
static inline s32 nla_get_s32(const struct nlattr *nla)
{
        return *(s32 *) nla_data(nla);
}

/**
 * nla_get_s32_default - return payload of s32 attribute or default
 * @nla: s32 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline s32 nla_get_s32_default(const struct nlattr *nla, s32 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_s32(nla);
}

/**
 * nla_get_s16 - return payload of s16 attribute
 * @nla: s16 netlink attribute
 */
static inline s16 nla_get_s16(const struct nlattr *nla)
{
        return *(s16 *) nla_data(nla);
}

/**
 * nla_get_s16_default - return payload of s16 attribute or default
 * @nla: s16 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline s16 nla_get_s16_default(const struct nlattr *nla, s16 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_s16(nla);
}

/**
 * nla_get_s8 - return payload of s8 attribute
 * @nla: s8 netlink attribute
 */
static inline s8 nla_get_s8(const struct nlattr *nla)
{
        return *(s8 *) nla_data(nla);
}

/**
 * nla_get_s8_default - return payload of s8 attribute or default
 * @nla: s8 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline s8 nla_get_s8_default(const struct nlattr *nla, s8 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_s8(nla);
}

/**
 * nla_get_s64 - return payload of s64 attribute
 * @nla: s64 netlink attribute
 */
static inline s64 nla_get_s64(const struct nlattr *nla)
{
        s64 tmp;

        nla_memcpy(&tmp, nla, sizeof(tmp));

        return tmp;
}

/**
 * nla_get_s64_default - return payload of s64 attribute or default
 * @nla: s64 netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline s64 nla_get_s64_default(const struct nlattr *nla, s64 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_s64(nla);
}

/**
 * nla_get_sint - return payload of uint attribute
 * @nla: uint netlink attribute
 */
static inline s64 nla_get_sint(const struct nlattr *nla)
{
        if (nla_len(nla) == sizeof(s32))
                return nla_get_s32(nla);
        return nla_get_s64(nla);
}

/**
 * nla_get_sint_default - return payload of sint attribute or default
 * @nla: sint netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline s64 nla_get_sint_default(const struct nlattr *nla, s64 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_sint(nla);
}

/**
 * nla_get_flag - return payload of flag attribute
 * @nla: flag netlink attribute
 */
static inline int nla_get_flag(const struct nlattr *nla)
{
        return !!nla;
}

/**
 * nla_get_msecs - return payload of msecs attribute
 * @nla: msecs netlink attribute
 *
 * Returns the number of milliseconds in jiffies.
 */
static inline unsigned long nla_get_msecs(const struct nlattr *nla)
{
        u64 msecs = nla_get_u64(nla);

        return msecs_to_jiffies((unsigned long) msecs);
}

/**
 * nla_get_msecs_default - return payload of msecs attribute or default
 * @nla: msecs netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline unsigned long nla_get_msecs_default(const struct nlattr *nla,
                                                  unsigned long defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_msecs(nla);
}

/**
 * nla_get_in_addr - return payload of IPv4 address attribute
 * @nla: IPv4 address netlink attribute
 */
static inline __be32 nla_get_in_addr(const struct nlattr *nla)
{
        return *(__be32 *) nla_data(nla);
}

/**
 * nla_get_in_addr_default - return payload of be32 attribute or default
 * @nla: IPv4 address netlink attribute, may be %NULL
 * @defvalue: default value to use if @nla is %NULL
 *
 * Return: the value of the attribute, or the default value if not present
 */
static inline __be32 nla_get_in_addr_default(const struct nlattr *nla,
                                             __be32 defvalue)
{
        if (!nla)
                return defvalue;
        return nla_get_in_addr(nla);
}

/**
 * nla_get_in6_addr - return payload of IPv6 address attribute
 * @nla: IPv6 address netlink attribute
 */
static inline struct in6_addr nla_get_in6_addr(const struct nlattr *nla)
{
        struct in6_addr tmp;

        nla_memcpy(&tmp, nla, sizeof(tmp));
        return tmp;
}

/**
 * nla_get_bitfield32 - return payload of 32 bitfield attribute
 * @nla: nla_bitfield32 attribute
 */
static inline struct nla_bitfield32 nla_get_bitfield32(const struct nlattr *nla)
{
        struct nla_bitfield32 tmp;

        nla_memcpy(&tmp, nla, sizeof(tmp));
        return tmp;
}

/**
 * nla_memdup - duplicate attribute memory (kmemdup)
 * @src: netlink attribute to duplicate from
 * @gfp: GFP mask
 */
static inline void *nla_memdup_noprof(const struct nlattr *src, gfp_t gfp)
{
        return kmemdup_noprof(nla_data(src), nla_len(src), gfp);
}
#define nla_memdup(...)        alloc_hooks(nla_memdup_noprof(__VA_ARGS__))

/**
 * nla_nest_start_noflag - Start a new level of nested attributes
 * @skb: socket buffer to add attributes to
 * @attrtype: attribute type of container
 *
 * This function exists for backward compatibility to use in APIs which never
 * marked their nest attributes with NLA_F_NESTED flag. New APIs should use
 * nla_nest_start() which sets the flag.
 *
 * Returns the container attribute or NULL on error
 */
static inline struct nlattr *nla_nest_start_noflag(struct sk_buff *skb,
                                                   int attrtype)
{
        struct nlattr *start = (struct nlattr *)skb_tail_pointer(skb);

        if (nla_put(skb, attrtype, 0, NULL) < 0)
                return NULL;

        return start;
}

/**
 * nla_nest_start - Start a new level of nested attributes, with NLA_F_NESTED
 * @skb: socket buffer to add attributes to
 * @attrtype: attribute type of container
 *
 * Unlike nla_nest_start_noflag(), mark the nest attribute with NLA_F_NESTED
 * flag. This is the preferred function to use in new code.
 *
 * Returns the container attribute or NULL on error
 */
static inline struct nlattr *nla_nest_start(struct sk_buff *skb, int attrtype)
{
        return nla_nest_start_noflag(skb, attrtype | NLA_F_NESTED);
}

/**
 * nla_nest_end - Finalize nesting of attributes
 * @skb: socket buffer the attributes are stored in
 * @start: container attribute
 *
 * Corrects the container attribute header to include the all
 * appended attributes.
 *
 * Returns the total data length of the skb.
 */
static inline int nla_nest_end(struct sk_buff *skb, struct nlattr *start)
{
        start->nla_len = skb_tail_pointer(skb) - (unsigned char *)start;
        return skb->len;
}

/**
 * nla_nest_cancel - Cancel nesting of attributes
 * @skb: socket buffer the message is stored in
 * @start: container attribute
 *
 * Removes the container attribute and including all nested
 * attributes. Returns -EMSGSIZE
 */
static inline void nla_nest_cancel(struct sk_buff *skb, struct nlattr *start)
{
        nlmsg_trim(skb, start);
}

/**
 * __nla_validate_nested - Validate a stream of nested attributes
 * @start: container attribute
 * @maxtype: maximum attribute type to be expected
 * @policy: validation policy
 * @validate: validation strictness
 * @extack: extended ACK report struct
 *
 * Validates all attributes in the nested attribute stream against the
 * specified policy. Attributes with a type exceeding maxtype will be
 * ignored. See documentation of struct nla_policy for more details.
 *
 * Returns 0 on success or a negative error code.
 */
static inline int __nla_validate_nested(const struct nlattr *start, int maxtype,
                                        const struct nla_policy *policy,
                                        unsigned int validate,
                                        struct netlink_ext_ack *extack)
{
        return __nla_validate(nla_data(start), nla_len(start), maxtype, policy,
                              validate, extack);
}

static inline int
nla_validate_nested(const struct nlattr *start, int maxtype,
                    const struct nla_policy *policy,
                    struct netlink_ext_ack *extack)
{
        return __nla_validate_nested(start, maxtype, policy,
                                     NL_VALIDATE_STRICT, extack);
}

static inline int
nla_validate_nested_deprecated(const struct nlattr *start, int maxtype,
                               const struct nla_policy *policy,
                               struct netlink_ext_ack *extack)
{
        return __nla_validate_nested(start, maxtype, policy,
                                     NL_VALIDATE_LIBERAL, extack);
}

/**
 * nla_need_padding_for_64bit - test 64-bit alignment of the next attribute
 * @skb: socket buffer the message is stored in
 *
 * Return true if padding is needed to align the next attribute (nla_data()) to
 * a 64-bit aligned area.
 */
static inline bool nla_need_padding_for_64bit(struct sk_buff *skb)
{
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
        /* The nlattr header is 4 bytes in size, that's why we test
         * if the skb->data _is_ aligned.  A NOP attribute, plus
         * nlattr header for next attribute, will make nla_data()
         * 8-byte aligned.
         */
        if (IS_ALIGNED((unsigned long)skb_tail_pointer(skb), 8))
                return true;
#endif
        return false;
}

/**
 * nla_align_64bit - 64-bit align the nla_data() of next attribute
 * @skb: socket buffer the message is stored in
 * @padattr: attribute type for the padding
 *
 * Conditionally emit a padding netlink attribute in order to make
 * the next attribute we emit have a 64-bit aligned nla_data() area.
 * This will only be done in architectures which do not have
 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS defined.
 *
 * Returns zero on success or a negative error code.
 */
static inline int nla_align_64bit(struct sk_buff *skb, int padattr)
{
        if (nla_need_padding_for_64bit(skb) &&
            !nla_reserve(skb, padattr, 0))
                return -EMSGSIZE;

        return 0;
}

/**
 * nla_total_size_64bit - total length of attribute including padding
 * @payload: length of payload
 */
static inline int nla_total_size_64bit(int payload)
{
        return NLA_ALIGN(nla_attr_size(payload))
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
                + NLA_ALIGN(nla_attr_size(0))
#endif
                ;
}

/**
 * nla_for_each_attr - iterate over a stream of attributes
 * @pos: loop counter, set to current attribute
 * @head: head of attribute stream
 * @len: length of attribute stream
 * @rem: initialized to len, holds bytes currently remaining in stream
 */
#define nla_for_each_attr(pos, head, len, rem) \
        for (pos = head, rem = len; \
             nla_ok(pos, rem); \
             pos = nla_next(pos, &(rem)))

/**
 * nla_for_each_attr_type - iterate over a stream of attributes
 * @pos: loop counter, set to current attribute
 * @type: required attribute type for @pos
 * @head: head of attribute stream
 * @len: length of attribute stream
 * @rem: initialized to len, holds bytes currently remaining in stream
 */
#define nla_for_each_attr_type(pos, type, head, len, rem) \
        nla_for_each_attr(pos, head, len, rem) \
                if (nla_type(pos) == type)

/**
 * nla_for_each_nested - iterate over nested attributes
 * @pos: loop counter, set to current attribute
 * @nla: attribute containing the nested attributes
 * @rem: initialized to len, holds bytes currently remaining in stream
 */
#define nla_for_each_nested(pos, nla, rem) \
        nla_for_each_attr(pos, nla_data(nla), nla_len(nla), rem)

/**
 * nla_for_each_nested_type - iterate over nested attributes
 * @pos: loop counter, set to current attribute
 * @type: required attribute type for @pos
 * @nla: attribute containing the nested attributes
 * @rem: initialized to len, holds bytes currently remaining in stream
 */
#define nla_for_each_nested_type(pos, type, nla, rem) \
        nla_for_each_nested(pos, nla, rem) \
                if (nla_type(pos) == type)

/**
 * nla_is_last - Test if attribute is last in stream
 * @nla: attribute to test
 * @rem: bytes remaining in stream
 */
static inline bool nla_is_last(const struct nlattr *nla, int rem)
{
        return nla->nla_len == rem;
}

void nla_get_range_unsigned(const struct nla_policy *pt,
                            struct netlink_range_validation *range);
void nla_get_range_signed(const struct nla_policy *pt,
                          struct netlink_range_validation_signed *range);

struct netlink_policy_dump_state;

int netlink_policy_dump_add_policy(struct netlink_policy_dump_state **pstate,
                                   const struct nla_policy *policy,
                                   unsigned int maxtype);
int netlink_policy_dump_get_policy_idx(struct netlink_policy_dump_state *state,
                                       const struct nla_policy *policy,
                                       unsigned int maxtype);
bool netlink_policy_dump_loop(struct netlink_policy_dump_state *state);
int netlink_policy_dump_write(struct sk_buff *skb,
                              struct netlink_policy_dump_state *state);
int netlink_policy_dump_attr_size_estimate(const struct nla_policy *pt);
int netlink_policy_dump_write_attr(struct sk_buff *skb,
                                   const struct nla_policy *pt,
                                   int nestattr);
void netlink_policy_dump_free(struct netlink_policy_dump_state *state);

#endif



















































































    5 
    5 
    5 

    5 


    5 








    5 
    5 












    5 













    5 




    5 
    2 



    5 


    5 



    5 








    5 



    5 




    5 

    5 




    5 



    5 



    5 








    5 









    5 

    5 
    5 








    5 

























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
// SPDX-License-Identifier: GPL-2.0
/*
 * Helper routines for building identity mapping page tables. This is
 * included by both the compressed kernel and the regular kernel.
 */

static void free_pte(struct x86_mapping_info *info, pmd_t *pmd)
{
        pte_t *pte = pte_offset_kernel(pmd, 0);

        info->free_pgt_page(pte, info->context);
}

static void free_pmd(struct x86_mapping_info *info, pud_t *pud)
{
        pmd_t *pmd = pmd_offset(pud, 0);
        int i;

        for (i = 0; i < PTRS_PER_PMD; i++) {
                if (!pmd_present(pmd[i]))
                        continue;

                if (pmd_leaf(pmd[i]))
                        continue;

                free_pte(info, &pmd[i]);
        }

        info->free_pgt_page(pmd, info->context);
}

static void free_pud(struct x86_mapping_info *info, p4d_t *p4d)
{
        pud_t *pud = pud_offset(p4d, 0);
        int i;

        for (i = 0; i < PTRS_PER_PUD; i++) {
                if (!pud_present(pud[i]))
                        continue;

                if (pud_leaf(pud[i]))
                        continue;

                free_pmd(info, &pud[i]);
        }

        info->free_pgt_page(pud, info->context);
}

static void free_p4d(struct x86_mapping_info *info, pgd_t *pgd)
{
        p4d_t *p4d = p4d_offset(pgd, 0);
        int i;

        for (i = 0; i < PTRS_PER_P4D; i++) {
                if (!p4d_present(p4d[i]))
                        continue;

                free_pud(info, &p4d[i]);
        }

        if (pgtable_l5_enabled())
                info->free_pgt_page(p4d, info->context);
}

void kernel_ident_mapping_free(struct x86_mapping_info *info, pgd_t *pgd)
{
        int i;

        for (i = 0; i < PTRS_PER_PGD; i++) {
                if (!pgd_present(pgd[i]))
                        continue;

                free_p4d(info, &pgd[i]);
        }

        info->free_pgt_page(pgd, info->context);
}

static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,
                           unsigned long addr, unsigned long end)
{
        addr &= PMD_MASK;
        for (; addr < end; addr += PMD_SIZE) {
                pmd_t *pmd = pmd_page + pmd_index(addr);

                if (pmd_present(*pmd))
                        continue;

                set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag));
        }
}

static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
                          unsigned long addr, unsigned long end)
{
        unsigned long next;

        for (; addr < end; addr = next) {
                pud_t *pud = pud_page + pud_index(addr);
                pmd_t *pmd;
                bool use_gbpage;

                next = (addr & PUD_MASK) + PUD_SIZE;
                if (next > end)
                        next = end;

                /* if this is already a gbpage, this portion is already mapped */
                if (pud_leaf(*pud))
                        continue;

                /* Is using a gbpage allowed? */
                use_gbpage = info->direct_gbpages;

                /* Don't use gbpage if it maps more than the requested region. */
                /* at the begining: */
                use_gbpage &= ((addr & ~PUD_MASK) == 0);
                /* ... or at the end: */
                use_gbpage &= ((next & ~PUD_MASK) == 0);

                /* Never overwrite existing mappings */
                use_gbpage &= !pud_present(*pud);

                if (use_gbpage) {
                        pud_t pudval;

                        pudval = __pud((addr - info->offset) | info->page_flag);
                        set_pud(pud, pudval);
                        continue;
                }

                if (pud_present(*pud)) {
                        pmd = pmd_offset(pud, 0);
                        ident_pmd_init(info, pmd, addr, next);
                        continue;
                }
                pmd = (pmd_t *)info->alloc_pgt_page(info->context);
                if (!pmd)
                        return -ENOMEM;
                ident_pmd_init(info, pmd, addr, next);
                set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag));
        }

        return 0;
}

static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page,
                          unsigned long addr, unsigned long end)
{
        unsigned long next;
        int result;

        for (; addr < end; addr = next) {
                p4d_t *p4d = p4d_page + p4d_index(addr);
                pud_t *pud;

                next = (addr & P4D_MASK) + P4D_SIZE;
                if (next > end)
                        next = end;

                if (p4d_present(*p4d)) {
                        pud = pud_offset(p4d, 0);
                        result = ident_pud_init(info, pud, addr, next);
                        if (result)
                                return result;

                        continue;
                }
                pud = (pud_t *)info->alloc_pgt_page(info->context);
                if (!pud)
                        return -ENOMEM;

                result = ident_pud_init(info, pud, addr, next);
                if (result)
                        return result;

                set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag | _PAGE_NOPTISHADOW));
        }

        return 0;
}

int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
                              unsigned long pstart, unsigned long pend)
{
        unsigned long addr = pstart + info->offset;
        unsigned long end = pend + info->offset;
        unsigned long next;
        int result;

        /* Set the default pagetable flags if not supplied */
        if (!info->kernpg_flag)
                info->kernpg_flag = _KERNPG_TABLE;

        /* Filter out unsupported __PAGE_KERNEL_* bits: */
        info->kernpg_flag &= __default_kernel_pte_mask;

        for (; addr < end; addr = next) {
                pgd_t *pgd = pgd_page + pgd_index(addr);
                p4d_t *p4d;

                next = (addr & PGDIR_MASK) + PGDIR_SIZE;
                if (next > end)
                        next = end;

                if (pgd_present(*pgd)) {
                        p4d = p4d_offset(pgd, 0);
                        result = ident_p4d_init(info, p4d, addr, next);
                        if (result)
                                return result;
                        continue;
                }

                p4d = (p4d_t *)info->alloc_pgt_page(info->context);
                if (!p4d)
                        return -ENOMEM;
                result = ident_p4d_init(info, p4d, addr, next);
                if (result)
                        return result;
                if (pgtable_l5_enabled()) {
                        set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag | _PAGE_NOPTISHADOW));
                } else {
                        /*
                         * With p4d folded, pgd is equal to p4d.
                         * The pgd entry has to point to the pud page table in this case.
                         */
                        pud_t *pud = pud_offset(p4d, 0);
                        set_pgd(pgd, __pgd(__pa(pud) | info->kernpg_flag | _PAGE_NOPTISHADOW));
                }
        }

        return 0;
}






























    1 

    2 













































































































   75 











  735 











    3 




































































  229 


  736 
  730 

  432 


  313 





   22 




   11 







  735 
  734 






  736 






























  715 
  717 




  187 



  200 
  168 
  248 










































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_NETFILTER_H
#define __LINUX_NETFILTER_H

#include <linux/init.h>
#include <linux/skbuff.h>
#include <linux/net.h>
#include <linux/if.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/wait.h>
#include <linux/list.h>
#include <linux/static_key.h>
#include <linux/module.h>
#include <linux/netfilter_defs.h>
#include <linux/netdevice.h>
#include <linux/sockptr.h>
#include <net/net_namespace.h>

static inline int NF_DROP_GETERR(int verdict)
{
        return -(verdict >> NF_VERDICT_QBITS);
}

static __always_inline int
NF_DROP_REASON(struct sk_buff *skb, enum skb_drop_reason reason, u32 err)
{
        BUILD_BUG_ON(err > 0xffff);

        kfree_skb_reason(skb, reason);

        return ((err << 16) | NF_STOLEN);
}

static inline int nf_inet_addr_cmp(const union nf_inet_addr *a1,
                                   const union nf_inet_addr *a2)
{
#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
        const unsigned long *ul1 = (const unsigned long *)a1;
        const unsigned long *ul2 = (const unsigned long *)a2;

        return ((ul1[0] ^ ul2[0]) | (ul1[1] ^ ul2[1])) == 0UL;
#else
        return a1->all[0] == a2->all[0] &&
               a1->all[1] == a2->all[1] &&
               a1->all[2] == a2->all[2] &&
               a1->all[3] == a2->all[3];
#endif
}

static inline void nf_inet_addr_mask(const union nf_inet_addr *a1,
                                     union nf_inet_addr *result,
                                     const union nf_inet_addr *mask)
{
#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
        const unsigned long *ua = (const unsigned long *)a1;
        unsigned long *ur = (unsigned long *)result;
        const unsigned long *um = (const unsigned long *)mask;

        ur[0] = ua[0] & um[0];
        ur[1] = ua[1] & um[1];
#else
        result->all[0] = a1->all[0] & mask->all[0];
        result->all[1] = a1->all[1] & mask->all[1];
        result->all[2] = a1->all[2] & mask->all[2];
        result->all[3] = a1->all[3] & mask->all[3];
#endif
}

int netfilter_init(void);

struct sk_buff;

struct nf_hook_ops;

struct sock;

struct nf_hook_state {
        u8 hook;
        u8 pf;
        struct net_device *in;
        struct net_device *out;
        struct sock *sk;
        struct net *net;
        int (*okfn)(struct net *, struct sock *, struct sk_buff *);
};

typedef unsigned int nf_hookfn(void *priv,
                               struct sk_buff *skb,
                               const struct nf_hook_state *state);
enum nf_hook_ops_type {
        NF_HOOK_OP_UNDEFINED,
        NF_HOOK_OP_NF_TABLES,
        NF_HOOK_OP_BPF,
};

struct nf_hook_ops {
        /* User fills in from here down. */
        nf_hookfn                *hook;
        struct net_device        *dev;
        void                        *priv;
        u8                        pf;
        enum nf_hook_ops_type        hook_ops_type:8;
        unsigned int                hooknum;
        /* Hooks are ordered in ascending priority. */
        int                        priority;
};

struct nf_hook_entry {
        nf_hookfn                        *hook;
        void                                *priv;
};

struct nf_hook_entries_rcu_head {
        struct rcu_head head;
        void        *allocation;
};

struct nf_hook_entries {
        u16                                num_hook_entries;
        /* padding */
        struct nf_hook_entry                hooks[];

        /* trailer: pointers to original orig_ops of each hook,
         * followed by rcu_head and scratch space used for freeing
         * the structure via call_rcu.
         *
         *   This is not part of struct nf_hook_entry since its only
         *   needed in slow path (hook register/unregister):
         * const struct nf_hook_ops     *orig_ops[]
         *
         *   For the same reason, we store this at end -- its
         *   only needed when a hook is deleted, not during
         *   packet path processing:
         * struct nf_hook_entries_rcu_head     head
         */
};

#ifdef CONFIG_NETFILTER
static inline struct nf_hook_ops **nf_hook_entries_get_hook_ops(const struct nf_hook_entries *e)
{
        unsigned int n = e->num_hook_entries;
        const void *hook_end;

        hook_end = &e->hooks[n]; /* this is *past* ->hooks[]! */

        return (struct nf_hook_ops **)hook_end;
}

static inline int
nf_hook_entry_hookfn(const struct nf_hook_entry *entry, struct sk_buff *skb,
                     struct nf_hook_state *state)
{
        return entry->hook(entry->priv, skb, state);
}

static inline void nf_hook_state_init(struct nf_hook_state *p,
                                      unsigned int hook,
                                      u_int8_t pf,
                                      struct net_device *indev,
                                      struct net_device *outdev,
                                      struct sock *sk,
                                      struct net *net,
                                      int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{
        p->hook = hook;
        p->pf = pf;
        p->in = indev;
        p->out = outdev;
        p->sk = sk;
        p->net = net;
        p->okfn = okfn;
}



struct nf_sockopt_ops {
        struct list_head list;

        u_int8_t pf;

        /* Non-inclusive ranges: use 0/0/NULL to never get called. */
        int set_optmin;
        int set_optmax;
        int (*set)(struct sock *sk, int optval, sockptr_t arg,
                   unsigned int len);
        int get_optmin;
        int get_optmax;
        int (*get)(struct sock *sk, int optval, void __user *user, int *len);
        /* Use the module struct to lock set/get code in place */
        struct module *owner;
};

/* Function to register/unregister hook points. */
int nf_register_net_hook(struct net *net, const struct nf_hook_ops *ops);
void nf_unregister_net_hook(struct net *net, const struct nf_hook_ops *ops);
int nf_register_net_hooks(struct net *net, const struct nf_hook_ops *reg,
                          unsigned int n);
void nf_unregister_net_hooks(struct net *net, const struct nf_hook_ops *reg,
                             unsigned int n);

/* Functions to register get/setsockopt ranges (non-inclusive).  You
   need to check permissions yourself! */
int nf_register_sockopt(struct nf_sockopt_ops *reg);
void nf_unregister_sockopt(struct nf_sockopt_ops *reg);

#ifdef CONFIG_JUMP_LABEL
extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
#endif

int nf_hook_slow(struct sk_buff *skb, struct nf_hook_state *state,
                 const struct nf_hook_entries *e, unsigned int i);

void nf_hook_slow_list(struct list_head *head, struct nf_hook_state *state,
                       const struct nf_hook_entries *e);
/**
 *        nf_hook - call a netfilter hook
 *
 *        Returns 1 if the hook has allowed the packet to pass.  The function
 *        okfn must be invoked by the caller in this case.  Any other return
 *        value indicates the packet has been consumed by the hook.
 */
static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
                          struct sock *sk, struct sk_buff *skb,
                          struct net_device *indev, struct net_device *outdev,
                          int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{
        struct nf_hook_entries *hook_head = NULL;
        int ret = 1;

#ifdef CONFIG_JUMP_LABEL
        if (__builtin_constant_p(pf) &&
            __builtin_constant_p(hook) &&
            !static_key_false(&nf_hooks_needed[pf][hook]))
                return 1;
#endif

        rcu_read_lock();
        switch (pf) {
        case NFPROTO_IPV4:
                hook_head = rcu_dereference(net->nf.hooks_ipv4[hook]);
                break;
        case NFPROTO_IPV6:
                hook_head = rcu_dereference(net->nf.hooks_ipv6[hook]);
                break;
        case NFPROTO_ARP:
#ifdef CONFIG_NETFILTER_FAMILY_ARP
                if (WARN_ON_ONCE(hook >= ARRAY_SIZE(net->nf.hooks_arp)))
                        break;
                hook_head = rcu_dereference(net->nf.hooks_arp[hook]);
#endif
                break;
        case NFPROTO_BRIDGE:
#ifdef CONFIG_NETFILTER_FAMILY_BRIDGE
                hook_head = rcu_dereference(net->nf.hooks_bridge[hook]);
#endif
                break;
        default:
                WARN_ON_ONCE(1);
                break;
        }

        if (hook_head) {
                struct nf_hook_state state;

                nf_hook_state_init(&state, hook, pf, indev, outdev,
                                   sk, net, okfn);

                ret = nf_hook_slow(skb, &state, hook_head, 0);
        }
        rcu_read_unlock();

        return ret;
}

/* Activate hook; either okfn or kfree_skb called, unless a hook
   returns NF_STOLEN (in which case, it's up to the hook to deal with
   the consequences).

   Returns -ERRNO if packet dropped.  Zero means queued, stolen or
   accepted.
*/

/* RR:
   > I don't want nf_hook to return anything because people might forget
   > about async and trust the return value to mean "packet was ok".

   AK:
   Just document it clearly, then you can expect some sense from kernel
   coders :)
*/

static inline int
NF_HOOK_COND(uint8_t pf, unsigned int hook, struct net *net, struct sock *sk,
             struct sk_buff *skb, struct net_device *in, struct net_device *out,
             int (*okfn)(struct net *, struct sock *, struct sk_buff *),
             bool cond)
{
        int ret;

        if (!cond ||
            ((ret = nf_hook(pf, hook, net, sk, skb, in, out, okfn)) == 1))
                ret = okfn(net, sk, skb);
        return ret;
}

static inline int
NF_HOOK(uint8_t pf, unsigned int hook, struct net *net, struct sock *sk, struct sk_buff *skb,
        struct net_device *in, struct net_device *out,
        int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{
        int ret = nf_hook(pf, hook, net, sk, skb, in, out, okfn);
        if (ret == 1)
                ret = okfn(net, sk, skb);
        return ret;
}

static inline void
NF_HOOK_LIST(uint8_t pf, unsigned int hook, struct net *net, struct sock *sk,
             struct list_head *head, struct net_device *in, struct net_device *out,
             int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{
        struct nf_hook_entries *hook_head = NULL;

#ifdef CONFIG_JUMP_LABEL
        if (__builtin_constant_p(pf) &&
            __builtin_constant_p(hook) &&
            !static_key_false(&nf_hooks_needed[pf][hook]))
                return;
#endif

        rcu_read_lock();
        switch (pf) {
        case NFPROTO_IPV4:
                hook_head = rcu_dereference(net->nf.hooks_ipv4[hook]);
                break;
        case NFPROTO_IPV6:
                hook_head = rcu_dereference(net->nf.hooks_ipv6[hook]);
                break;
        default:
                WARN_ON_ONCE(1);
                break;
        }

        if (hook_head) {
                struct nf_hook_state state;

                nf_hook_state_init(&state, hook, pf, in, out, sk, net, okfn);

                nf_hook_slow_list(head, &state, hook_head);
        }
        rcu_read_unlock();
}

/* Call setsockopt() */
int nf_setsockopt(struct sock *sk, u_int8_t pf, int optval, sockptr_t opt,
                  unsigned int len);
int nf_getsockopt(struct sock *sk, u_int8_t pf, int optval, char __user *opt,
                  int *len);

struct flowi;
struct nf_queue_entry;

__sum16 nf_checksum(struct sk_buff *skb, unsigned int hook,
                    unsigned int dataoff, u_int8_t protocol,
                    unsigned short family);

__sum16 nf_checksum_partial(struct sk_buff *skb, unsigned int hook,
                            unsigned int dataoff, unsigned int len,
                            u_int8_t protocol, unsigned short family);
int nf_route(struct net *net, struct dst_entry **dst, struct flowi *fl,
             bool strict, unsigned short family);

#include <net/flow.h>

struct nf_conn;
enum nf_nat_manip_type;
struct nlattr;

struct nf_nat_hook {
        int (*parse_nat_setup)(struct nf_conn *ct, enum nf_nat_manip_type manip,
                               const struct nlattr *attr);
        void (*decode_session)(struct sk_buff *skb, struct flowi *fl);
        void (*remove_nat_bysrc)(struct nf_conn *ct);
};

extern const struct nf_nat_hook __rcu *nf_nat_hook;

static inline void
nf_nat_decode_session(struct sk_buff *skb, struct flowi *fl, u_int8_t family)
{
#if IS_ENABLED(CONFIG_NF_NAT)
        const struct nf_nat_hook *nat_hook;

        rcu_read_lock();
        nat_hook = rcu_dereference(nf_nat_hook);
        if (nat_hook && nat_hook->decode_session)
                nat_hook->decode_session(skb, fl);
        rcu_read_unlock();
#endif
}

#else /* !CONFIG_NETFILTER */
static inline int
NF_HOOK_COND(uint8_t pf, unsigned int hook, struct net *net, struct sock *sk,
             struct sk_buff *skb, struct net_device *in, struct net_device *out,
             int (*okfn)(struct net *, struct sock *, struct sk_buff *),
             bool cond)
{
        return okfn(net, sk, skb);
}

static inline int
NF_HOOK(uint8_t pf, unsigned int hook, struct net *net, struct sock *sk,
        struct sk_buff *skb, struct net_device *in, struct net_device *out,
        int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{
        return okfn(net, sk, skb);
}

static inline void
NF_HOOK_LIST(uint8_t pf, unsigned int hook, struct net *net, struct sock *sk,
             struct list_head *head, struct net_device *in, struct net_device *out,
             int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{
        /* nothing to do */
}

static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
                          struct sock *sk, struct sk_buff *skb,
                          struct net_device *indev, struct net_device *outdev,
                          int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{
        return 1;
}
struct flowi;
static inline void
nf_nat_decode_session(struct sk_buff *skb, struct flowi *fl, u_int8_t family)
{
}
#endif /*CONFIG_NETFILTER*/

#if IS_ENABLED(CONFIG_NF_CONNTRACK)
#include <linux/netfilter/nf_conntrack_zones_common.h>

void nf_ct_attach(struct sk_buff *, const struct sk_buff *);
void nf_ct_set_closing(struct nf_conntrack *nfct);
struct nf_conntrack_tuple;
bool nf_ct_get_tuple_skb(struct nf_conntrack_tuple *dst_tuple,
                         const struct sk_buff *skb);
#else
static inline void nf_ct_attach(struct sk_buff *new, struct sk_buff *skb) {}
static inline void nf_ct_set_closing(struct nf_conntrack *nfct) {}
struct nf_conntrack_tuple;
static inline bool nf_ct_get_tuple_skb(struct nf_conntrack_tuple *dst_tuple,
                                       const struct sk_buff *skb)
{
        return false;
}
#endif

struct nf_conn;
enum ip_conntrack_info;

struct nf_ct_hook {
        int (*update)(struct net *net, struct sk_buff *skb);
        void (*destroy)(struct nf_conntrack *);
        bool (*get_tuple_skb)(struct nf_conntrack_tuple *,
                              const struct sk_buff *);
        void (*attach)(struct sk_buff *nskb, const struct sk_buff *skb);
        void (*set_closing)(struct nf_conntrack *nfct);
        int (*confirm)(struct sk_buff *skb);
};
extern const struct nf_ct_hook __rcu *nf_ct_hook;

struct nlattr;

struct nfnl_ct_hook {
        size_t (*build_size)(const struct nf_conn *ct);
        int (*build)(struct sk_buff *skb, struct nf_conn *ct,
                     enum ip_conntrack_info ctinfo,
                     u_int16_t ct_attr, u_int16_t ct_info_attr);
        int (*parse)(const struct nlattr *attr, struct nf_conn *ct);
        int (*attach_expect)(const struct nlattr *attr, struct nf_conn *ct,
                             u32 portid, u32 report);
        void (*seq_adjust)(struct sk_buff *skb, struct nf_conn *ct,
                           enum ip_conntrack_info ctinfo, s32 off);
};
extern const struct nfnl_ct_hook __rcu *nfnl_ct_hook;

struct nf_defrag_hook {
        struct module *owner;
        int (*enable)(struct net *net);
        void (*disable)(struct net *net);
};

extern const struct nf_defrag_hook __rcu *nf_defrag_v4_hook;
extern const struct nf_defrag_hook __rcu *nf_defrag_v6_hook;

/*
 * nf_skb_duplicated - TEE target has sent a packet
 *
 * When a xtables target sends a packet, the OUTPUT and POSTROUTING
 * hooks are traversed again, i.e. nft and xtables are invoked recursively.
 *
 * This is used by xtables TEE target to prevent the duplicated skb from
 * being duplicated again.
 */
DECLARE_PER_CPU(bool, nf_skb_duplicated);

/*
 * Contains bitmask of ctnetlink event subscribers, if any.
 * Can't be pernet due to NETLINK_LISTEN_ALL_NSID setsockopt flag.
 */
extern u8 nf_ctnetlink_has_listener;
#endif /*__LINUX_NETFILTER_H*/





































































































































   10 








   23 





   23 












   23 
   23 


















   23 


   23 

   23 









  132 
  132 










   10 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
// SPDX-License-Identifier: GPL-2.0
/*
 * Wakeup statistics in sysfs
 *
 * Copyright (c) 2019 Linux Foundation
 * Copyright (c) 2019 Greg Kroah-Hartman <gregkh@linuxfoundation.org>
 * Copyright (c) 2019 Google Inc.
 */

#include <linux/device.h>
#include <linux/idr.h>
#include <linux/init.h>
#include <linux/kdev_t.h>
#include <linux/kernel.h>
#include <linux/kobject.h>
#include <linux/slab.h>
#include <linux/timekeeping.h>

#include "power.h"

static struct class *wakeup_class;

#define wakeup_attr(_name)                                                \
static ssize_t _name##_show(struct device *dev,                                \
                            struct device_attribute *attr, char *buf)        \
{                                                                        \
        struct wakeup_source *ws = dev_get_drvdata(dev);                \
                                                                        \
        return sysfs_emit(buf, "%lu\n", ws->_name);                        \
}                                                                        \
static DEVICE_ATTR_RO(_name)

wakeup_attr(active_count);
wakeup_attr(event_count);
wakeup_attr(wakeup_count);
wakeup_attr(expire_count);

static ssize_t active_time_ms_show(struct device *dev,
                                   struct device_attribute *attr, char *buf)
{
        struct wakeup_source *ws = dev_get_drvdata(dev);
        ktime_t active_time =
                ws->active ? ktime_sub(ktime_get(), ws->last_time) : 0;

        return sysfs_emit(buf, "%lld\n", ktime_to_ms(active_time));
}
static DEVICE_ATTR_RO(active_time_ms);

static ssize_t total_time_ms_show(struct device *dev,
                                  struct device_attribute *attr, char *buf)
{
        struct wakeup_source *ws = dev_get_drvdata(dev);
        ktime_t active_time;
        ktime_t total_time = ws->total_time;

        if (ws->active) {
                active_time = ktime_sub(ktime_get(), ws->last_time);
                total_time = ktime_add(total_time, active_time);
        }

        return sysfs_emit(buf, "%lld\n", ktime_to_ms(total_time));
}
static DEVICE_ATTR_RO(total_time_ms);

static ssize_t max_time_ms_show(struct device *dev,
                                struct device_attribute *attr, char *buf)
{
        struct wakeup_source *ws = dev_get_drvdata(dev);
        ktime_t active_time;
        ktime_t max_time = ws->max_time;

        if (ws->active) {
                active_time = ktime_sub(ktime_get(), ws->last_time);
                if (active_time > max_time)
                        max_time = active_time;
        }

        return sysfs_emit(buf, "%lld\n", ktime_to_ms(max_time));
}
static DEVICE_ATTR_RO(max_time_ms);

static ssize_t last_change_ms_show(struct device *dev,
                                   struct device_attribute *attr, char *buf)
{
        struct wakeup_source *ws = dev_get_drvdata(dev);

        return sysfs_emit(buf, "%lld\n", ktime_to_ms(ws->last_time));
}
static DEVICE_ATTR_RO(last_change_ms);

static ssize_t name_show(struct device *dev, struct device_attribute *attr,
                         char *buf)
{
        struct wakeup_source *ws = dev_get_drvdata(dev);

        return sysfs_emit(buf, "%s\n", ws->name);
}
static DEVICE_ATTR_RO(name);

static ssize_t prevent_suspend_time_ms_show(struct device *dev,
                                            struct device_attribute *attr,
                                            char *buf)
{
        struct wakeup_source *ws = dev_get_drvdata(dev);
        ktime_t prevent_sleep_time = ws->prevent_sleep_time;

        if (ws->active && ws->autosleep_enabled) {
                prevent_sleep_time = ktime_add(prevent_sleep_time,
                        ktime_sub(ktime_get(), ws->start_prevent_time));
        }

        return sysfs_emit(buf, "%lld\n", ktime_to_ms(prevent_sleep_time));
}
static DEVICE_ATTR_RO(prevent_suspend_time_ms);

static struct attribute *wakeup_source_attrs[] = {
        &dev_attr_name.attr,
        &dev_attr_active_count.attr,
        &dev_attr_event_count.attr,
        &dev_attr_wakeup_count.attr,
        &dev_attr_expire_count.attr,
        &dev_attr_active_time_ms.attr,
        &dev_attr_total_time_ms.attr,
        &dev_attr_max_time_ms.attr,
        &dev_attr_last_change_ms.attr,
        &dev_attr_prevent_suspend_time_ms.attr,
        NULL,
};
ATTRIBUTE_GROUPS(wakeup_source);

static void device_create_release(struct device *dev)
{
        kfree(dev);
}

static struct device *wakeup_source_device_create(struct device *parent,
                                                  struct wakeup_source *ws)
{
        struct device *dev = NULL;
        int retval;

        dev = kzalloc(sizeof(*dev), GFP_KERNEL);
        if (!dev) {
                retval = -ENOMEM;
                goto error;
        }

        device_initialize(dev);
        dev->devt = MKDEV(0, 0);
        dev->class = wakeup_class;
        dev->parent = parent;
        dev->groups = wakeup_source_groups;
        dev->release = device_create_release;
        dev_set_drvdata(dev, ws);
        device_set_pm_not_required(dev);

        retval = dev_set_name(dev, "wakeup%d", ws->id);
        if (retval)
                goto error;

        retval = device_add(dev);
        if (retval)
                goto error;

        return dev;

error:
        put_device(dev);
        return ERR_PTR(retval);
}

/**
 * wakeup_source_sysfs_add - Add wakeup_source attributes to sysfs.
 * @parent: Device given wakeup source is associated with (or NULL if virtual).
 * @ws: Wakeup source to be added in sysfs.
 */
int wakeup_source_sysfs_add(struct device *parent, struct wakeup_source *ws)
{
        struct device *dev;

        dev = wakeup_source_device_create(parent, ws);
        if (IS_ERR(dev))
                return PTR_ERR(dev);
        ws->dev = dev;

        return 0;
}

/**
 * pm_wakeup_source_sysfs_add - Add wakeup_source attributes to sysfs
 * for a device if they're missing.
 * @parent: Device given wakeup source is associated with
 */
int pm_wakeup_source_sysfs_add(struct device *parent)
{
        if (!parent->power.wakeup || parent->power.wakeup->dev)
                return 0;

        return wakeup_source_sysfs_add(parent, parent->power.wakeup);
}

/**
 * wakeup_source_sysfs_remove - Remove wakeup_source attributes from sysfs.
 * @ws: Wakeup source to be removed from sysfs.
 */
void wakeup_source_sysfs_remove(struct wakeup_source *ws)
{
        device_unregister(ws->dev);
}

static int __init wakeup_sources_sysfs_init(void)
{
        wakeup_class = class_create("wakeup");

        return PTR_ERR_OR_ZERO(wakeup_class);
}
postcore_initcall(wakeup_sources_sysfs_init);












































  181 
  197 









  126 










  149 




  206 










   51 

  224 



























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_POLL_H
#define _LINUX_POLL_H


#include <linux/compiler.h>
#include <linux/ktime.h>
#include <linux/wait.h>
#include <linux/string.h>
#include <linux/fs.h>
#include <linux/uaccess.h>
#include <uapi/linux/poll.h>
#include <uapi/linux/eventpoll.h>

/* ~832 bytes of stack space used max in sys_select/sys_poll before allocating
   additional memory. */
#define MAX_STACK_ALLOC 832
#define FRONTEND_STACK_ALLOC        256
#define SELECT_STACK_ALLOC        FRONTEND_STACK_ALLOC
#define POLL_STACK_ALLOC        FRONTEND_STACK_ALLOC
#define WQUEUES_STACK_ALLOC        (MAX_STACK_ALLOC - FRONTEND_STACK_ALLOC)
#define N_INLINE_POLL_ENTRIES        (WQUEUES_STACK_ALLOC / sizeof(struct poll_table_entry))

#define DEFAULT_POLLMASK (EPOLLIN | EPOLLOUT | EPOLLRDNORM | EPOLLWRNORM)

struct poll_table_struct;

/* 
 * structures and helpers for f_op->poll implementations
 */
typedef void (*poll_queue_proc)(struct file *, wait_queue_head_t *, struct poll_table_struct *);

/*
 * Do not touch the structure directly, use the access functions
 * poll_does_not_wait() and poll_requested_events() instead.
 */
typedef struct poll_table_struct {
        poll_queue_proc _qproc;
        __poll_t _key;
} poll_table;

static inline void poll_wait(struct file * filp, wait_queue_head_t * wait_address, poll_table *p)
{
        if (p && p->_qproc && wait_address)
                p->_qproc(filp, wait_address, p);
}

/*
 * Return true if it is guaranteed that poll will not wait. This is the case
 * if the poll() of another file descriptor in the set got an event, so there
 * is no need for waiting.
 */
static inline bool poll_does_not_wait(const poll_table *p)
{
        return p == NULL || p->_qproc == NULL;
}

/*
 * Return the set of events that the application wants to poll for.
 * This is useful for drivers that need to know whether a DMA transfer has
 * to be started implicitly on poll(). You typically only want to do that
 * if the application is actually polling for POLLIN and/or POLLOUT.
 */
static inline __poll_t poll_requested_events(const poll_table *p)
{
        return p ? p->_key : ~(__poll_t)0;
}

static inline void init_poll_funcptr(poll_table *pt, poll_queue_proc qproc)
{
        pt->_qproc = qproc;
        pt->_key   = ~(__poll_t)0; /* all events enabled */
}

static inline bool file_can_poll(struct file *file)
{
        return file->f_op->poll;
}

static inline __poll_t vfs_poll(struct file *file, struct poll_table_struct *pt)
{
        if (unlikely(!file->f_op->poll))
                return DEFAULT_POLLMASK;
        return file->f_op->poll(file, pt);
}

struct poll_table_entry {
        struct file *filp;
        __poll_t key;
        wait_queue_entry_t wait;
        wait_queue_head_t *wait_address;
};

/*
 * Structures and helpers for select/poll syscall
 */
struct poll_wqueues {
        poll_table pt;
        struct poll_table_page *table;
        struct task_struct *polling_task;
        int triggered;
        int error;
        int inline_index;
        struct poll_table_entry inline_entries[N_INLINE_POLL_ENTRIES];
};

extern void poll_initwait(struct poll_wqueues *pwq);
extern void poll_freewait(struct poll_wqueues *pwq);
extern u64 select_estimate_accuracy(struct timespec64 *tv);

#define MAX_INT64_SECONDS (((s64)(~((u64)0)>>1)/HZ)-1)

extern int core_sys_select(int n, fd_set __user *inp, fd_set __user *outp,
                           fd_set __user *exp, struct timespec64 *end_time);

extern int poll_select_set_timeout(struct timespec64 *to, time64_t sec,
                                   long nsec);

#define __MAP(v, from, to) \
        (from < to ? (v & from) * (to/from) : (v & from) / (from/to))

static inline __u16 mangle_poll(__poll_t val)
{
        __u16 v = (__force __u16)val;
#define M(X) __MAP(v, (__force __u16)EPOLL##X, POLL##X)
        return M(IN) | M(OUT) | M(PRI) | M(ERR) | M(NVAL) |
                M(RDNORM) | M(RDBAND) | M(WRNORM) | M(WRBAND) |
                M(HUP) | M(RDHUP) | M(MSG);
#undef M
}

static inline __poll_t demangle_poll(u16 val)
{
#define M(X) (__force __poll_t)__MAP(val, POLL##X, (__force __u16)EPOLL##X)
        return M(IN) | M(OUT) | M(PRI) | M(ERR) | M(NVAL) |
                M(RDNORM) | M(RDBAND) | M(WRNORM) | M(WRBAND) |
                M(HUP) | M(RDHUP) | M(MSG);
#undef M
}
#undef __MAP


#endif /* _LINUX_POLL_H */











































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (C) B.A.T.M.A.N. contributors:
 *
 * Marek Lindner, Simon Wunderlich
 */

#ifndef _NET_BATMAN_ADV_ORIGINATOR_H_
#define _NET_BATMAN_ADV_ORIGINATOR_H_

#include "main.h"

#include <linux/compiler.h>
#include <linux/if_ether.h>
#include <linux/jhash.h>
#include <linux/kref.h>
#include <linux/netlink.h>
#include <linux/skbuff.h>
#include <linux/types.h>

bool batadv_compare_orig(const struct hlist_node *node, const void *data2);
int batadv_originator_init(struct batadv_priv *bat_priv);
void batadv_originator_free(struct batadv_priv *bat_priv);
void batadv_purge_orig_ref(struct batadv_priv *bat_priv);
void batadv_orig_node_release(struct kref *ref);
struct batadv_orig_node *batadv_orig_node_new(struct batadv_priv *bat_priv,
                                              const u8 *addr);
struct batadv_hardif_neigh_node *
batadv_hardif_neigh_get(const struct batadv_hard_iface *hard_iface,
                        const u8 *neigh_addr);
void batadv_hardif_neigh_release(struct kref *ref);
struct batadv_neigh_node *
batadv_neigh_node_get_or_create(struct batadv_orig_node *orig_node,
                                struct batadv_hard_iface *hard_iface,
                                const u8 *neigh_addr);
void batadv_neigh_node_release(struct kref *ref);
struct batadv_neigh_node *
batadv_orig_router_get(struct batadv_orig_node *orig_node,
                       const struct batadv_hard_iface *if_outgoing);
struct batadv_neigh_node *
batadv_orig_to_router(struct batadv_priv *bat_priv, u8 *orig_addr,
                      struct batadv_hard_iface *if_outgoing);
struct batadv_neigh_ifinfo *
batadv_neigh_ifinfo_new(struct batadv_neigh_node *neigh,
                        struct batadv_hard_iface *if_outgoing);
struct batadv_neigh_ifinfo *
batadv_neigh_ifinfo_get(struct batadv_neigh_node *neigh,
                        struct batadv_hard_iface *if_outgoing);
void batadv_neigh_ifinfo_release(struct kref *ref);

int batadv_hardif_neigh_dump(struct sk_buff *msg, struct netlink_callback *cb);

struct batadv_orig_ifinfo *
batadv_orig_ifinfo_get(struct batadv_orig_node *orig_node,
                       struct batadv_hard_iface *if_outgoing);
struct batadv_orig_ifinfo *
batadv_orig_ifinfo_new(struct batadv_orig_node *orig_node,
                       struct batadv_hard_iface *if_outgoing);
void batadv_orig_ifinfo_release(struct kref *ref);

int batadv_orig_dump(struct sk_buff *msg, struct netlink_callback *cb);
struct batadv_orig_node_vlan *
batadv_orig_node_vlan_new(struct batadv_orig_node *orig_node,
                          unsigned short vid);
struct batadv_orig_node_vlan *
batadv_orig_node_vlan_get(struct batadv_orig_node *orig_node,
                          unsigned short vid);
void batadv_orig_node_vlan_release(struct kref *ref);

/**
 * batadv_choose_orig() - Return the index of the orig entry in the hash table
 * @data: mac address of the originator node
 * @size: the size of the hash table
 *
 * Return: the hash index where the object represented by @data should be
 * stored at.
 */
static inline u32 batadv_choose_orig(const void *data, u32 size)
{
        u32 hash = 0;

        hash = jhash(data, ETH_ALEN, hash);
        return hash % size;
}

struct batadv_orig_node *
batadv_orig_hash_find(struct batadv_priv *bat_priv, const void *data);

/**
 * batadv_orig_node_vlan_put() - decrement the refcounter and possibly release
 *  the originator-vlan object
 * @orig_vlan: the originator-vlan object to release
 */
static inline void
batadv_orig_node_vlan_put(struct batadv_orig_node_vlan *orig_vlan)
{
        if (!orig_vlan)
                return;

        kref_put(&orig_vlan->refcount, batadv_orig_node_vlan_release);
}

/**
 * batadv_neigh_ifinfo_put() - decrement the refcounter and possibly release
 *  the neigh_ifinfo
 * @neigh_ifinfo: the neigh_ifinfo object to release
 */
static inline void
batadv_neigh_ifinfo_put(struct batadv_neigh_ifinfo *neigh_ifinfo)
{
        if (!neigh_ifinfo)
                return;

        kref_put(&neigh_ifinfo->refcount, batadv_neigh_ifinfo_release);
}

/**
 * batadv_hardif_neigh_put() - decrement the hardif neighbors refcounter
 *  and possibly release it
 * @hardif_neigh: hardif neigh neighbor to free
 */
static inline void
batadv_hardif_neigh_put(struct batadv_hardif_neigh_node *hardif_neigh)
{
        if (!hardif_neigh)
                return;

        kref_put(&hardif_neigh->refcount, batadv_hardif_neigh_release);
}

/**
 * batadv_neigh_node_put() - decrement the neighbors refcounter and possibly
 *  release it
 * @neigh_node: neigh neighbor to free
 */
static inline void batadv_neigh_node_put(struct batadv_neigh_node *neigh_node)
{
        if (!neigh_node)
                return;

        kref_put(&neigh_node->refcount, batadv_neigh_node_release);
}

/**
 * batadv_orig_ifinfo_put() - decrement the refcounter and possibly release
 *  the orig_ifinfo
 * @orig_ifinfo: the orig_ifinfo object to release
 */
static inline void
batadv_orig_ifinfo_put(struct batadv_orig_ifinfo *orig_ifinfo)
{
        if (!orig_ifinfo)
                return;

        kref_put(&orig_ifinfo->refcount, batadv_orig_ifinfo_release);
}

/**
 * batadv_orig_node_put() - decrement the orig node refcounter and possibly
 *  release it
 * @orig_node: the orig node to free
 */
static inline void batadv_orig_node_put(struct batadv_orig_node *orig_node)
{
        if (!orig_node)
                return;

        kref_put(&orig_node->refcount, batadv_orig_node_release);
}

#endif /* _NET_BATMAN_ADV_ORIGINATOR_H_ */














    2 









































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM page_pool

#if !defined(_TRACE_PAGE_POOL_H) || defined(TRACE_HEADER_MULTI_READ)
#define      _TRACE_PAGE_POOL_H

#include <linux/types.h>
#include <linux/tracepoint.h>

#include <trace/events/mmflags.h>
#include <net/page_pool/types.h>

TRACE_EVENT(page_pool_release,

        TP_PROTO(const struct page_pool *pool,
                 s32 inflight, u32 hold, u32 release),

        TP_ARGS(pool, inflight, hold, release),

        TP_STRUCT__entry(
                __field(const struct page_pool *, pool)
                __field(s32,        inflight)
                __field(u32,        hold)
                __field(u32,        release)
                __field(u64,        cnt)
        ),

        TP_fast_assign(
                __entry->pool                = pool;
                __entry->inflight        = inflight;
                __entry->hold                = hold;
                __entry->release        = release;
                __entry->cnt                = pool->destroy_cnt;
        ),

        TP_printk("page_pool=%p inflight=%d hold=%u release=%u cnt=%llu",
                __entry->pool, __entry->inflight, __entry->hold,
                __entry->release, __entry->cnt)
);

TRACE_EVENT(page_pool_state_release,

        TP_PROTO(const struct page_pool *pool,
                 netmem_ref netmem, u32 release),

        TP_ARGS(pool, netmem, release),

        TP_STRUCT__entry(
                __field(const struct page_pool *,        pool)
                __field(unsigned long,                        netmem)
                __field(u32,                                release)
                __field(unsigned long,                        pfn)
        ),

        TP_fast_assign(
                __entry->pool                = pool;
                __entry->netmem                = (__force unsigned long)netmem;
                __entry->release        = release;
                __entry->pfn                = netmem_pfn_trace(netmem);
        ),

        TP_printk("page_pool=%p netmem=%p is_net_iov=%lu pfn=0x%lx release=%u",
                  __entry->pool, (void *)__entry->netmem,
                  __entry->netmem & NET_IOV, __entry->pfn, __entry->release)
);

TRACE_EVENT(page_pool_state_hold,

        TP_PROTO(const struct page_pool *pool,
                 netmem_ref netmem, u32 hold),

        TP_ARGS(pool, netmem, hold),

        TP_STRUCT__entry(
                __field(const struct page_pool *,        pool)
                __field(unsigned long,                        netmem)
                __field(u32,                                hold)
                __field(unsigned long,                        pfn)
        ),

        TP_fast_assign(
                __entry->pool        = pool;
                __entry->netmem        = (__force unsigned long)netmem;
                __entry->hold        = hold;
                __entry->pfn        = netmem_pfn_trace(netmem);
        ),

        TP_printk("page_pool=%p netmem=%p is_net_iov=%lu, pfn=0x%lx hold=%u",
                  __entry->pool, (void *)__entry->netmem,
                  __entry->netmem & NET_IOV, __entry->pfn, __entry->hold)
);

TRACE_EVENT(page_pool_update_nid,

        TP_PROTO(const struct page_pool *pool, int new_nid),

        TP_ARGS(pool, new_nid),

        TP_STRUCT__entry(
                __field(const struct page_pool *, pool)
                __field(int,                          pool_nid)
                __field(int,                          new_nid)
        ),

        TP_fast_assign(
                __entry->pool                = pool;
                __entry->pool_nid        = pool->p.nid;
                __entry->new_nid        = new_nid;
        ),

        TP_printk("page_pool=%p pool_nid=%d new_nid=%d",
                  __entry->pool, __entry->pool_nid, __entry->new_nid)
);

#endif /* _TRACE_PAGE_POOL_H */

/* This part must be outside protection */
#include <trace/define_trace.h>






































































































    4 
    4 




    4 

    3 

    3 

    3 




    3 



    3 







    1 
    4 






















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/drivers/base/map.c
 *
 * (C) Copyright Al Viro 2002,2003
 *
 * NOTE: data structure needs to be changed.  It works, but for large dev_t
 * it will be too slow.  It is isolated, though, so these changes will be
 * local to that file.
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/mutex.h>
#include <linux/kdev_t.h>
#include <linux/kobject.h>
#include <linux/kobj_map.h>

struct kobj_map {
        struct probe {
                struct probe *next;
                dev_t dev;
                unsigned long range;
                struct module *owner;
                kobj_probe_t *get;
                int (*lock)(dev_t, void *);
                void *data;
        } *probes[255];
        struct mutex *lock;
};

int kobj_map(struct kobj_map *domain, dev_t dev, unsigned long range,
             struct module *module, kobj_probe_t *probe,
             int (*lock)(dev_t, void *), void *data)
{
        unsigned int n = MAJOR(dev + range - 1) - MAJOR(dev) + 1;
        unsigned int index = MAJOR(dev);
        unsigned int i;
        struct probe *p;

        if (n > 255)
                n = 255;

        p = kmalloc_array(n, sizeof(struct probe), GFP_KERNEL);
        if (p == NULL)
                return -ENOMEM;

        for (i = 0; i < n; i++, p++) {
                p->owner = module;
                p->get = probe;
                p->lock = lock;
                p->dev = dev;
                p->range = range;
                p->data = data;
        }
        mutex_lock(domain->lock);
        for (i = 0, p -= n; i < n; i++, p++, index++) {
                struct probe **s = &domain->probes[index % 255];
                while (*s && (*s)->range < range)
                        s = &(*s)->next;
                p->next = *s;
                *s = p;
        }
        mutex_unlock(domain->lock);
        return 0;
}

void kobj_unmap(struct kobj_map *domain, dev_t dev, unsigned long range)
{
        unsigned int n = MAJOR(dev + range - 1) - MAJOR(dev) + 1;
        unsigned int index = MAJOR(dev);
        unsigned int i;
        struct probe *found = NULL;

        if (n > 255)
                n = 255;

        mutex_lock(domain->lock);
        for (i = 0; i < n; i++, index++) {
                struct probe **s;
                for (s = &domain->probes[index % 255]; *s; s = &(*s)->next) {
                        struct probe *p = *s;
                        if (p->dev == dev && p->range == range) {
                                *s = p->next;
                                if (!found)
                                        found = p;
                                break;
                        }
                }
        }
        mutex_unlock(domain->lock);
        kfree(found);
}

struct kobject *kobj_lookup(struct kobj_map *domain, dev_t dev, int *index)
{
        struct kobject *kobj;
        struct probe *p;
        unsigned long best = ~0UL;

retry:
        mutex_lock(domain->lock);
        for (p = domain->probes[MAJOR(dev) % 255]; p; p = p->next) {
                struct kobject *(*probe)(dev_t, int *, void *);
                struct module *owner;
                void *data;

                if (p->dev > dev || p->dev + p->range - 1 < dev)
                        continue;
                if (p->range - 1 >= best)
                        break;
                if (!try_module_get(p->owner))
                        continue;
                owner = p->owner;
                data = p->data;
                probe = p->get;
                best = p->range - 1;
                *index = dev - p->dev;
                if (p->lock && p->lock(dev, data) < 0) {
                        module_put(owner);
                        continue;
                }
                mutex_unlock(domain->lock);
                kobj = probe(dev, index, data);
                /* Currently ->owner protects _only_ ->probe() itself. */
                module_put(owner);
                if (kobj)
                        return kobj;
                goto retry;
        }
        mutex_unlock(domain->lock);
        return NULL;
}

struct kobj_map *kobj_map_init(kobj_probe_t *base_probe, struct mutex *lock)
{
        struct kobj_map *p = kmalloc(sizeof(struct kobj_map), GFP_KERNEL);
        struct probe *base = kzalloc(sizeof(*base), GFP_KERNEL);
        int i;

        if ((p == NULL) || (base == NULL)) {
                kfree(p);
                kfree(base);
                return NULL;
        }

        base->dev = 1;
        base->range = ~0;
        base->get = base_probe;
        for (i = 0; i < 255; i++)
                p->probes[i] = base;
        p->lock = lock;
        return p;
}






































































































































   31 











   29 



   10 

   17 
   10 














   37 









   34 

























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Cryptographic API.
 *
 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
 * Copyright (c) 2005 Herbert Xu <herbert@gondor.apana.org.au>
 */
#ifndef _CRYPTO_INTERNAL_H
#define _CRYPTO_INTERNAL_H

#include <crypto/algapi.h>
#include <linux/completion.h>
#include <linux/err.h>
#include <linux/jump_label.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/notifier.h>
#include <linux/numa.h>
#include <linux/refcount.h>
#include <linux/rwsem.h>
#include <linux/scatterlist.h>
#include <linux/sched.h>
#include <linux/types.h>

struct crypto_instance;
struct crypto_template;

struct crypto_larval {
        struct crypto_alg alg;
        struct crypto_alg *adult;
        struct completion completion;
        u32 mask;
        bool test_started;
};

enum {
        CRYPTOA_UNSPEC,
        CRYPTOA_ALG,
        CRYPTOA_TYPE,
        __CRYPTOA_MAX,
};

#define CRYPTOA_MAX (__CRYPTOA_MAX - 1)

/* Maximum number of (rtattr) parameters for each template. */
#define CRYPTO_MAX_ATTRS 32

extern struct list_head crypto_alg_list;
extern struct rw_semaphore crypto_alg_sem;
extern struct blocking_notifier_head crypto_chain;

int alg_test(const char *driver, const char *alg, u32 type, u32 mask);

#if !IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) || \
    IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)
static inline bool crypto_boot_test_finished(void)
{
        return true;
}
static inline void set_crypto_boot_test_finished(void)
{
}
#else
DECLARE_STATIC_KEY_FALSE(__crypto_boot_test_finished);
static inline bool crypto_boot_test_finished(void)
{
        return static_branch_likely(&__crypto_boot_test_finished);
}
static inline void set_crypto_boot_test_finished(void)
{
        static_branch_enable(&__crypto_boot_test_finished);
}
#endif /* !IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) ||
        * IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)
        */

#ifdef CONFIG_PROC_FS
void __init crypto_init_proc(void);
void __exit crypto_exit_proc(void);
#else
static inline void crypto_init_proc(void)
{ }
static inline void crypto_exit_proc(void)
{ }
#endif

static inline unsigned int crypto_cipher_ctxsize(struct crypto_alg *alg)
{
        return alg->cra_ctxsize;
}

static inline unsigned int crypto_compress_ctxsize(struct crypto_alg *alg)
{
        return alg->cra_ctxsize;
}

struct crypto_alg *crypto_mod_get(struct crypto_alg *alg);
struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask);

struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask);
void crypto_schedule_test(struct crypto_larval *larval);
void crypto_alg_tested(const char *name, int err);

void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
                          struct crypto_alg *nalg);
void crypto_remove_final(struct list_head *list);
void crypto_shoot_alg(struct crypto_alg *alg);
struct crypto_tfm *__crypto_alloc_tfmgfp(struct crypto_alg *alg, u32 type,
                                         u32 mask, gfp_t gfp);
struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 type,
                                      u32 mask);
void *crypto_create_tfm_node(struct crypto_alg *alg,
                        const struct crypto_type *frontend, int node);
void *crypto_clone_tfm(const struct crypto_type *frontend,
                       struct crypto_tfm *otfm);

static inline void *crypto_create_tfm(struct crypto_alg *alg,
                        const struct crypto_type *frontend)
{
        return crypto_create_tfm_node(alg, frontend, NUMA_NO_NODE);
}

struct crypto_alg *crypto_find_alg(const char *alg_name,
                                   const struct crypto_type *frontend,
                                   u32 type, u32 mask);

void *crypto_alloc_tfm_node(const char *alg_name,
                       const struct crypto_type *frontend, u32 type, u32 mask,
                       int node);

static inline void *crypto_alloc_tfm(const char *alg_name,
                       const struct crypto_type *frontend, u32 type, u32 mask)
{
        return crypto_alloc_tfm_node(alg_name, frontend, type, mask, NUMA_NO_NODE);
}

int crypto_probing_notify(unsigned long val, void *v);

unsigned int crypto_alg_extsize(struct crypto_alg *alg);

int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
                        u32 type, u32 mask);

static inline struct crypto_alg *crypto_alg_get(struct crypto_alg *alg)
{
        refcount_inc(&alg->cra_refcnt);
        return alg;
}

static inline void crypto_alg_put(struct crypto_alg *alg)
{
        if (refcount_dec_and_test(&alg->cra_refcnt) && alg->cra_destroy)
                alg->cra_destroy(alg);
}

static inline int crypto_tmpl_get(struct crypto_template *tmpl)
{
        return try_module_get(tmpl->module);
}

static inline void crypto_tmpl_put(struct crypto_template *tmpl)
{
        module_put(tmpl->module);
}

static inline int crypto_is_larval(struct crypto_alg *alg)
{
        return alg->cra_flags & CRYPTO_ALG_LARVAL;
}

static inline int crypto_is_dead(struct crypto_alg *alg)
{
        return alg->cra_flags & CRYPTO_ALG_DEAD;
}

static inline int crypto_is_moribund(struct crypto_alg *alg)
{
        return alg->cra_flags & (CRYPTO_ALG_DEAD | CRYPTO_ALG_DYING);
}

static inline void crypto_notify(unsigned long val, void *v)
{
        blocking_notifier_call_chain(&crypto_chain, val, v);
}

static inline void crypto_yield(u32 flags)
{
        if (flags & CRYPTO_TFM_REQ_MAY_SLEEP)
                cond_resched();
}

static inline int crypto_is_test_larval(struct crypto_larval *larval)
{
        return larval->alg.cra_driver_name[0];
}

static inline struct crypto_tfm *crypto_tfm_get(struct crypto_tfm *tfm)
{
        return refcount_inc_not_zero(&tfm->refcnt) ? tfm : ERR_PTR(-EOVERFLOW);
}

#endif        /* _CRYPTO_INTERNAL_H */

















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __X86_KERNEL_FPU_LEGACY_H
#define __X86_KERNEL_FPU_LEGACY_H

#include <asm/fpu/types.h>

extern unsigned int mxcsr_feature_mask;

static inline void ldmxcsr(u32 mxcsr)
{
        asm volatile("ldmxcsr %0" :: "m" (mxcsr));
}

/*
 * Returns 0 on success or the trap number when the operation raises an
 * exception.
 */
#define user_insn(insn, output, input...)                                \
({                                                                        \
        int err;                                                        \
                                                                        \
        might_fault();                                                        \
                                                                        \
        asm volatile(ASM_STAC "\n"                                        \
                     "1: " #insn "\n"                                        \
                     "2: " ASM_CLAC "\n"                                \
                     _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_MCE_SAFE)        \
                     : [err] "=a" (err), output                                \
                     : "0"(0), input);                                        \
        err;                                                                \
})

#define kernel_insn_err(insn, output, input...)                                \
({                                                                        \
        int err;                                                        \
        asm volatile("1:" #insn "\n\t"                                        \
                     "2:\n"                                                \
                     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %[err]) \
                     : [err] "=r" (err), output                                \
                     : "0"(0), input);                                        \
        err;                                                                \
})

#define kernel_insn(insn, output, input...)                                \
        asm volatile("1:" #insn "\n\t"                                        \
                     "2:\n"                                                \
                     _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FPU_RESTORE)        \
                     : output : input)

static inline int fnsave_to_user_sigframe(struct fregs_state __user *fx)
{
        return user_insn(fnsave %[fx]; fwait,  [fx] "=m" (*fx), "m" (*fx));
}

static inline int fxsave_to_user_sigframe(struct fxregs_state __user *fx)
{
        if (IS_ENABLED(CONFIG_X86_32))
                return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx));
        else
                return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx));

}

static inline void fxrstor(struct fxregs_state *fx)
{
        if (IS_ENABLED(CONFIG_X86_32))
                kernel_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
        else
                kernel_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline int fxrstor_safe(struct fxregs_state *fx)
{
        if (IS_ENABLED(CONFIG_X86_32))
                return kernel_insn_err(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
        else
                return kernel_insn_err(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline int fxrstor_from_user_sigframe(struct fxregs_state __user *fx)
{
        if (IS_ENABLED(CONFIG_X86_32))
                return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
        else
                return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline void frstor(struct fregs_state *fx)
{
        kernel_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline int frstor_safe(struct fregs_state *fx)
{
        return kernel_insn_err(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline int frstor_from_user_sigframe(struct fregs_state __user *fx)
{
        return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline void fxsave(struct fxregs_state *fx)
{
        if (IS_ENABLED(CONFIG_X86_32))
                asm volatile( "fxsave %[fx]" : [fx] "=m" (*fx));
        else
                asm volatile("fxsaveq %[fx]" : [fx] "=m" (*fx));
}

#endif























































































  760 

















   46 














    5 











































   25 









  327 























































    2 














    1 






















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TIMEKEEPING_H
#define _LINUX_TIMEKEEPING_H

#include <linux/errno.h>
#include <linux/clocksource_ids.h>
#include <linux/ktime.h>

/* Included from linux/ktime.h */

void timekeeping_init(void);
extern int timekeeping_suspended;

/* Architecture timer tick functions: */
extern void legacy_timer_tick(unsigned long ticks);

/*
 * Get and set timeofday
 */
extern int do_settimeofday64(const struct timespec64 *ts);
extern int do_sys_settimeofday64(const struct timespec64 *tv,
                                 const struct timezone *tz);

/*
 * ktime_get() family - read the current time in a multitude of ways.
 *
 * The default time reference is CLOCK_MONOTONIC, starting at
 * boot time but not counting the time spent in suspend.
 * For other references, use the functions with "real", "clocktai",
 * "boottime" and "raw" suffixes.
 *
 * To get the time in a different format, use the ones with
 * "ns", "ts64" and "seconds" suffix.
 *
 * See Documentation/core-api/timekeeping.rst for more details.
 */


/*
 * timespec64 based interfaces
 */
extern void ktime_get_raw_ts64(struct timespec64 *ts);
extern void ktime_get_ts64(struct timespec64 *ts);
extern void ktime_get_real_ts64(struct timespec64 *tv);
extern void ktime_get_coarse_ts64(struct timespec64 *ts);
extern void ktime_get_coarse_real_ts64(struct timespec64 *ts);

/* Multigrain timestamp interfaces */
extern void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts);
extern void ktime_get_real_ts64_mg(struct timespec64 *ts);
extern unsigned long timekeeping_get_mg_floor_swaps(void);

void getboottime64(struct timespec64 *ts);

/*
 * time64_t base interfaces
 */
extern time64_t ktime_get_seconds(void);
extern time64_t __ktime_get_real_seconds(void);
extern time64_t ktime_get_real_seconds(void);

/*
 * ktime_t based interfaces
 */

enum tk_offsets {
        TK_OFFS_REAL,
        TK_OFFS_BOOT,
        TK_OFFS_TAI,
        TK_OFFS_MAX,
};

extern ktime_t ktime_get(void);
extern ktime_t ktime_get_with_offset(enum tk_offsets offs);
extern ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs);
extern ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs);
extern ktime_t ktime_get_raw(void);
extern u32 ktime_get_resolution_ns(void);

/**
 * ktime_get_real - get the real (wall-) time in ktime_t format
 *
 * Returns: real (wall) time in ktime_t format
 */
static inline ktime_t ktime_get_real(void)
{
        return ktime_get_with_offset(TK_OFFS_REAL);
}

static inline ktime_t ktime_get_coarse_real(void)
{
        return ktime_get_coarse_with_offset(TK_OFFS_REAL);
}

/**
 * ktime_get_boottime - Get monotonic time since boot in ktime_t format
 *
 * This is similar to CLOCK_MONTONIC/ktime_get, but also includes the
 * time spent in suspend.
 *
 * Returns: monotonic time since boot in ktime_t format
 */
static inline ktime_t ktime_get_boottime(void)
{
        return ktime_get_with_offset(TK_OFFS_BOOT);
}

static inline ktime_t ktime_get_coarse_boottime(void)
{
        return ktime_get_coarse_with_offset(TK_OFFS_BOOT);
}

/**
 * ktime_get_clocktai - Get the TAI time of day in ktime_t format
 *
 * Returns: the TAI time of day in ktime_t format
 */
static inline ktime_t ktime_get_clocktai(void)
{
        return ktime_get_with_offset(TK_OFFS_TAI);
}

static inline ktime_t ktime_get_coarse_clocktai(void)
{
        return ktime_get_coarse_with_offset(TK_OFFS_TAI);
}

static inline ktime_t ktime_get_coarse(void)
{
        struct timespec64 ts;

        ktime_get_coarse_ts64(&ts);
        return timespec64_to_ktime(ts);
}

static inline u64 ktime_get_coarse_ns(void)
{
        return ktime_to_ns(ktime_get_coarse());
}

static inline u64 ktime_get_coarse_real_ns(void)
{
        return ktime_to_ns(ktime_get_coarse_real());
}

static inline u64 ktime_get_coarse_boottime_ns(void)
{
        return ktime_to_ns(ktime_get_coarse_boottime());
}

static inline u64 ktime_get_coarse_clocktai_ns(void)
{
        return ktime_to_ns(ktime_get_coarse_clocktai());
}

/**
 * ktime_mono_to_real - Convert monotonic time to clock realtime
 * @mono: monotonic time to convert
 *
 * Returns: time converted to realtime clock
 */
static inline ktime_t ktime_mono_to_real(ktime_t mono)
{
        return ktime_mono_to_any(mono, TK_OFFS_REAL);
}

/**
 * ktime_get_ns - Get the current time in nanoseconds
 *
 * Returns: current time converted to nanoseconds
 */
static inline u64 ktime_get_ns(void)
{
        return ktime_to_ns(ktime_get());
}

/**
 * ktime_get_real_ns - Get the current real/wall time in nanoseconds
 *
 * Returns: current real time converted to nanoseconds
 */
static inline u64 ktime_get_real_ns(void)
{
        return ktime_to_ns(ktime_get_real());
}

/**
 * ktime_get_boottime_ns - Get the monotonic time since boot in nanoseconds
 *
 * Returns: current boottime converted to nanoseconds
 */
static inline u64 ktime_get_boottime_ns(void)
{
        return ktime_to_ns(ktime_get_boottime());
}

/**
 * ktime_get_clocktai_ns - Get the current TAI time of day in nanoseconds
 *
 * Returns: current TAI time converted to nanoseconds
 */
static inline u64 ktime_get_clocktai_ns(void)
{
        return ktime_to_ns(ktime_get_clocktai());
}

/**
 * ktime_get_raw_ns - Get the raw monotonic time in nanoseconds
 *
 * Returns: current raw monotonic time converted to nanoseconds
 */
static inline u64 ktime_get_raw_ns(void)
{
        return ktime_to_ns(ktime_get_raw());
}

extern u64 ktime_get_mono_fast_ns(void);
extern u64 ktime_get_raw_fast_ns(void);
extern u64 ktime_get_boot_fast_ns(void);
extern u64 ktime_get_tai_fast_ns(void);
extern u64 ktime_get_real_fast_ns(void);

/*
 * timespec64/time64_t interfaces utilizing the ktime based ones
 * for API completeness, these could be implemented more efficiently
 * if needed.
 */
static inline void ktime_get_boottime_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_boottime());
}

static inline void ktime_get_coarse_boottime_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_coarse_boottime());
}

static inline time64_t ktime_get_boottime_seconds(void)
{
        return ktime_divns(ktime_get_coarse_boottime(), NSEC_PER_SEC);
}

static inline void ktime_get_clocktai_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_clocktai());
}

static inline void ktime_get_coarse_clocktai_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_coarse_clocktai());
}

static inline time64_t ktime_get_clocktai_seconds(void)
{
        return ktime_divns(ktime_get_coarse_clocktai(), NSEC_PER_SEC);
}

/*
 * RTC specific
 */
extern bool timekeeping_rtc_skipsuspend(void);
extern bool timekeeping_rtc_skipresume(void);

extern void timekeeping_inject_sleeptime64(const struct timespec64 *delta);

/**
 * struct ktime_timestamps - Simultaneous mono/boot/real timestamps
 * @mono:        Monotonic timestamp
 * @boot:        Boottime timestamp
 * @real:        Realtime timestamp
 */
struct ktime_timestamps {
        u64                mono;
        u64                boot;
        u64                real;
};

/**
 * struct system_time_snapshot - simultaneous raw/real time capture with
 *                                 counter value
 * @cycles:        Clocksource counter value to produce the system times
 * @real:        Realtime system time
 * @boot:        Boot time
 * @raw:        Monotonic raw system time
 * @cs_id:        Clocksource ID
 * @clock_was_set_seq:        The sequence number of clock-was-set events
 * @cs_was_changed_seq:        The sequence number of clocksource change events
 */
struct system_time_snapshot {
        u64                        cycles;
        ktime_t                        real;
        ktime_t                        boot;
        ktime_t                        raw;
        enum clocksource_ids        cs_id;
        unsigned int                clock_was_set_seq;
        u8                        cs_was_changed_seq;
};

/**
 * struct system_device_crosststamp - system/device cross-timestamp
 *                                      (synchronized capture)
 * @device:                Device time
 * @sys_realtime:        Realtime simultaneous with device time
 * @sys_monoraw:        Monotonic raw simultaneous with device time
 */
struct system_device_crosststamp {
        ktime_t device;
        ktime_t sys_realtime;
        ktime_t sys_monoraw;
};

/**
 * struct system_counterval_t - system counter value with the ID of the
 *                                corresponding clocksource
 * @cycles:        System counter value
 * @cs_id:        Clocksource ID corresponding to system counter value. Used by
 *                timekeeping code to verify comparability of two cycle values.
 *                The default ID, CSID_GENERIC, does not identify a specific
 *                clocksource.
 * @use_nsecs:        @cycles is in nanoseconds.
 */
struct system_counterval_t {
        u64                        cycles;
        enum clocksource_ids        cs_id;
        bool                        use_nsecs;
};

extern bool ktime_real_to_base_clock(ktime_t treal,
                                     enum clocksource_ids base_id, u64 *cycles);
extern bool timekeeping_clocksource_has_base(enum clocksource_ids id);

/*
 * Get cross timestamp between system clock and device clock
 */
extern int get_device_system_crosststamp(
                        int (*get_time_fn)(ktime_t *device_time,
                                struct system_counterval_t *system_counterval,
                                void *ctx),
                        void *ctx,
                        struct system_time_snapshot *history,
                        struct system_device_crosststamp *xtstamp);

/*
 * Simultaneously snapshot realtime and monotonic raw clocks
 */
extern void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot);

/* NMI safe mono/boot/realtime timestamps */
extern void ktime_get_fast_timestamps(struct ktime_timestamps *snap);

/*
 * Persistent clock related interfaces
 */
extern int persistent_clock_is_local;

extern void read_persistent_clock64(struct timespec64 *ts);
void read_persistent_wall_and_boot_offset(struct timespec64 *wall_clock,
                                          struct timespec64 *boot_offset);
#ifdef CONFIG_GENERIC_CMOS_UPDATE
extern int update_persistent_clock64(struct timespec64 now);
#endif

#endif






















































   49 






































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* SCTP kernel reference Implementation
 * (C) Copyright IBM Corp. 2001, 2004
 * Copyright (c) 1999-2000 Cisco, Inc.
 * Copyright (c) 1999-2001 Motorola, Inc.
 * Copyright (c) 2001 Intel Corp.
 * Copyright (c) 2001 Nokia, Inc.
 * Copyright (c) 2001 La Monte H.P. Yarroll
 *
 * This file is part of the SCTP kernel reference Implementation
 *
 * Various protocol defined structures.
 *
 * Please send any bug reports or fixes you make to the
 * email address(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Or submit a bug report through the following website:
 *    http://www.sf.net/projects/lksctp
 *
 * Written or modified by:
 *    La Monte H.P. Yarroll <piggy@acm.org>
 *    Karl Knutson <karl@athena.chicago.il.us>
 *    Jon Grimm <jgrimm@us.ibm.com>
 *    Xingang Guo <xingang.guo@intel.com>
 *    randall@sctp.chicago.il.us
 *    kmorneau@cisco.com
 *    qxie1@email.mot.com
 *    Sridhar Samudrala <sri@us.ibm.com>
 *    Kevin Gao <kevin.gao@intel.com>
 *
 * Any bugs reported given to us we will try to fix... any fixes shared will
 * be incorporated into the next SCTP release.
 */
#ifndef __LINUX_SCTP_H__
#define __LINUX_SCTP_H__

#include <linux/in.h>                /* We need in_addr.  */
#include <linux/in6.h>                /* We need in6_addr.  */
#include <linux/skbuff.h>

#include <uapi/linux/sctp.h>

/* Section 3.1.  SCTP Common Header Format */
struct sctphdr {
        __be16 source;
        __be16 dest;
        __be32 vtag;
        __le32 checksum;
};

static inline struct sctphdr *sctp_hdr(const struct sk_buff *skb)
{
        return (struct sctphdr *)skb_transport_header(skb);
}

/* Section 3.2.  Chunk Field Descriptions. */
struct sctp_chunkhdr {
        __u8 type;
        __u8 flags;
        __be16 length;
};


/* Section 3.2.  Chunk Type Values.
 * [Chunk Type] identifies the type of information contained in the Chunk
 * Value field. It takes a value from 0 to 254. The value of 255 is
 * reserved for future use as an extension field.
 */
enum sctp_cid {
        SCTP_CID_DATA                        = 0,
        SCTP_CID_INIT                        = 1,
        SCTP_CID_INIT_ACK                = 2,
        SCTP_CID_SACK                        = 3,
        SCTP_CID_HEARTBEAT                = 4,
        SCTP_CID_HEARTBEAT_ACK                = 5,
        SCTP_CID_ABORT                        = 6,
        SCTP_CID_SHUTDOWN                = 7,
        SCTP_CID_SHUTDOWN_ACK                = 8,
        SCTP_CID_ERROR                        = 9,
        SCTP_CID_COOKIE_ECHO                = 10,
        SCTP_CID_COOKIE_ACK                = 11,
        SCTP_CID_ECN_ECNE                = 12,
        SCTP_CID_ECN_CWR                = 13,
        SCTP_CID_SHUTDOWN_COMPLETE        = 14,

        /* AUTH Extension Section 4.1 */
        SCTP_CID_AUTH                        = 0x0F,

        /* sctp ndata 5.1. I-DATA */
        SCTP_CID_I_DATA                        = 0x40,

        /* PR-SCTP Sec 3.2 */
        SCTP_CID_FWD_TSN                = 0xC0,

        /* Use hex, as defined in ADDIP sec. 3.1 */
        SCTP_CID_ASCONF                        = 0xC1,
        SCTP_CID_I_FWD_TSN                = 0xC2,
        SCTP_CID_ASCONF_ACK                = 0x80,
        SCTP_CID_RECONF                        = 0x82,
        SCTP_CID_PAD                        = 0x84,
}; /* enum */


/* Section 3.2
 *  Chunk Types are encoded such that the highest-order two bits specify
 *  the action that must be taken if the processing endpoint does not
 *  recognize the Chunk Type.
 */
enum {
        SCTP_CID_ACTION_DISCARD     = 0x00,
        SCTP_CID_ACTION_DISCARD_ERR = 0x40,
        SCTP_CID_ACTION_SKIP        = 0x80,
        SCTP_CID_ACTION_SKIP_ERR    = 0xc0,
};

enum { SCTP_CID_ACTION_MASK = 0xc0, };

/* This flag is used in Chunk Flags for ABORT and SHUTDOWN COMPLETE.
 *
 * 3.3.7 Abort Association (ABORT) (6):
 *    The T bit is set to 0 if the sender had a TCB that it destroyed.
 *    If the sender did not have a TCB it should set this bit to 1.
 */
enum { SCTP_CHUNK_FLAG_T = 0x01 };

/*
 *  Set the T bit
 *
 *      0                   1                   2                   3
 *      0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 *     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *     |   Type = 14   |Reserved     |T|      Length = 4               |
 *     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 * Chunk Flags: 8 bits
 *
 *   Reserved:  7 bits
 *     Set to 0 on transmit and ignored on receipt.
 *
 *   T bit:  1 bit
 *     The T bit is set to 0 if the sender had a TCB that it destroyed. If
 *     the sender did NOT have a TCB it should set this bit to 1.
 *
 * Note: Special rules apply to this chunk for verification, please
 * see Section 8.5.1 for details.
 */

#define sctp_test_T_bit(c)    ((c)->chunk_hdr->flags & SCTP_CHUNK_FLAG_T)

/* RFC 2960
 * Section 3.2.1 Optional/Variable-length Parmaeter Format.
 */

struct sctp_paramhdr {
        __be16 type;
        __be16 length;
};

enum sctp_param {

        /* RFC 2960 Section 3.3.5 */
        SCTP_PARAM_HEARTBEAT_INFO                = cpu_to_be16(1),
        /* RFC 2960 Section 3.3.2.1 */
        SCTP_PARAM_IPV4_ADDRESS                        = cpu_to_be16(5),
        SCTP_PARAM_IPV6_ADDRESS                        = cpu_to_be16(6),
        SCTP_PARAM_STATE_COOKIE                        = cpu_to_be16(7),
        SCTP_PARAM_UNRECOGNIZED_PARAMETERS        = cpu_to_be16(8),
        SCTP_PARAM_COOKIE_PRESERVATIVE                = cpu_to_be16(9),
        SCTP_PARAM_HOST_NAME_ADDRESS                = cpu_to_be16(11),
        SCTP_PARAM_SUPPORTED_ADDRESS_TYPES        = cpu_to_be16(12),
        SCTP_PARAM_ECN_CAPABLE                        = cpu_to_be16(0x8000),

        /* AUTH Extension Section 3 */
        SCTP_PARAM_RANDOM                        = cpu_to_be16(0x8002),
        SCTP_PARAM_CHUNKS                        = cpu_to_be16(0x8003),
        SCTP_PARAM_HMAC_ALGO                        = cpu_to_be16(0x8004),

        /* Add-IP: Supported Extensions, Section 4.2 */
        SCTP_PARAM_SUPPORTED_EXT        = cpu_to_be16(0x8008),

        /* PR-SCTP Sec 3.1 */
        SCTP_PARAM_FWD_TSN_SUPPORT        = cpu_to_be16(0xc000),

        /* Add-IP Extension. Section 3.2 */
        SCTP_PARAM_ADD_IP                = cpu_to_be16(0xc001),
        SCTP_PARAM_DEL_IP                = cpu_to_be16(0xc002),
        SCTP_PARAM_ERR_CAUSE                = cpu_to_be16(0xc003),
        SCTP_PARAM_SET_PRIMARY                = cpu_to_be16(0xc004),
        SCTP_PARAM_SUCCESS_REPORT        = cpu_to_be16(0xc005),
        SCTP_PARAM_ADAPTATION_LAYER_IND = cpu_to_be16(0xc006),

        /* RE-CONFIG. Section 4 */
        SCTP_PARAM_RESET_OUT_REQUEST                = cpu_to_be16(0x000d),
        SCTP_PARAM_RESET_IN_REQUEST                = cpu_to_be16(0x000e),
        SCTP_PARAM_RESET_TSN_REQUEST                = cpu_to_be16(0x000f),
        SCTP_PARAM_RESET_RESPONSE                = cpu_to_be16(0x0010),
        SCTP_PARAM_RESET_ADD_OUT_STREAMS        = cpu_to_be16(0x0011),
        SCTP_PARAM_RESET_ADD_IN_STREAMS                = cpu_to_be16(0x0012),
}; /* enum */


/* RFC 2960 Section 3.2.1
 *  The Parameter Types are encoded such that the highest-order two bits
 *  specify the action that must be taken if the processing endpoint does
 *  not recognize the Parameter Type.
 *
 */
enum {
        SCTP_PARAM_ACTION_DISCARD     = cpu_to_be16(0x0000),
        SCTP_PARAM_ACTION_DISCARD_ERR = cpu_to_be16(0x4000),
        SCTP_PARAM_ACTION_SKIP        = cpu_to_be16(0x8000),
        SCTP_PARAM_ACTION_SKIP_ERR    = cpu_to_be16(0xc000),
};

enum { SCTP_PARAM_ACTION_MASK = cpu_to_be16(0xc000), };

/* RFC 2960 Section 3.3.1 Payload Data (DATA) (0) */

struct sctp_datahdr {
        __be32 tsn;
        __be16 stream;
        __be16 ssn;
        __u32 ppid;
        /* __u8  payload[]; */
};

struct sctp_data_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_datahdr data_hdr;
};

struct sctp_idatahdr {
        __be32 tsn;
        __be16 stream;
        __be16 reserved;
        __be32 mid;
        union {
                __u32 ppid;
                __be32 fsn;
        };
        __u8 payload[0];
};

struct sctp_idata_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_idatahdr data_hdr;
};

/* DATA Chuck Specific Flags */
enum {
        SCTP_DATA_MIDDLE_FRAG        = 0x00,
        SCTP_DATA_LAST_FRAG        = 0x01,
        SCTP_DATA_FIRST_FRAG        = 0x02,
        SCTP_DATA_NOT_FRAG        = 0x03,
        SCTP_DATA_UNORDERED        = 0x04,
        SCTP_DATA_SACK_IMM        = 0x08,
};
enum { SCTP_DATA_FRAG_MASK = 0x03, };


/* RFC 2960 Section 3.3.2 Initiation (INIT) (1)
 *
 *  This chunk is used to initiate a SCTP association between two
 *  endpoints.
 */
struct sctp_inithdr {
        __be32 init_tag;
        __be32 a_rwnd;
        __be16 num_outbound_streams;
        __be16 num_inbound_streams;
        __be32 initial_tsn;
        /* __u8  params[]; */
};

struct sctp_init_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_inithdr init_hdr;
};


/* Section 3.3.2.1. IPv4 Address Parameter (5) */
struct sctp_ipv4addr_param {
        struct sctp_paramhdr param_hdr;
        struct in_addr addr;
};

/* Section 3.3.2.1. IPv6 Address Parameter (6) */
struct sctp_ipv6addr_param {
        struct sctp_paramhdr param_hdr;
        struct in6_addr addr;
};

/* Section 3.3.2.1 Cookie Preservative (9) */
struct sctp_cookie_preserve_param {
        struct sctp_paramhdr param_hdr;
        __be32 lifespan_increment;
};

/* Section 3.3.2.1 Host Name Address (11) */
struct sctp_hostname_param {
        struct sctp_paramhdr param_hdr;
        uint8_t hostname[];
};

/* Section 3.3.2.1 Supported Address Types (12) */
struct sctp_supported_addrs_param {
        struct sctp_paramhdr param_hdr;
        __be16 types[];
};

/* ADDIP Section 3.2.6 Adaptation Layer Indication */
struct sctp_adaptation_ind_param {
        struct sctp_paramhdr param_hdr;
        __be32 adaptation_ind;
};

/* ADDIP Section 4.2.7 Supported Extensions Parameter */
struct sctp_supported_ext_param {
        struct sctp_paramhdr param_hdr;
        __u8 chunks[];
};

/* AUTH Section 3.1 Random */
struct sctp_random_param {
        struct sctp_paramhdr param_hdr;
        __u8 random_val[];
};

/* AUTH Section 3.2 Chunk List */
struct sctp_chunks_param {
        struct sctp_paramhdr param_hdr;
        __u8 chunks[];
};

/* AUTH Section 3.3 HMAC Algorithm */
struct sctp_hmac_algo_param {
        struct sctp_paramhdr param_hdr;
        __be16 hmac_ids[];
};

/* RFC 2960.  Section 3.3.3 Initiation Acknowledgement (INIT ACK) (2):
 *   The INIT ACK chunk is used to acknowledge the initiation of an SCTP
 *   association.
 */
struct sctp_initack_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_inithdr init_hdr;
};

/* Section 3.3.3.1 State Cookie (7) */
struct sctp_cookie_param {
        struct sctp_paramhdr p;
        __u8 body[];
};

/* Section 3.3.3.1 Unrecognized Parameters (8) */
struct sctp_unrecognized_param {
        struct sctp_paramhdr param_hdr;
        struct sctp_paramhdr unrecognized;
};



/*
 * 3.3.4 Selective Acknowledgement (SACK) (3):
 *
 *  This chunk is sent to the peer endpoint to acknowledge received DATA
 *  chunks and to inform the peer endpoint of gaps in the received
 *  subsequences of DATA chunks as represented by their TSNs.
 */

struct sctp_gap_ack_block {
        __be16 start;
        __be16 end;
};

union sctp_sack_variable {
        struct sctp_gap_ack_block gab;
        __be32 dup;
};

struct sctp_sackhdr {
        __be32 cum_tsn_ack;
        __be32 a_rwnd;
        __be16 num_gap_ack_blocks;
        __be16 num_dup_tsns;
        /* union sctp_sack_variable variable[]; */
};

struct sctp_sack_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_sackhdr sack_hdr;
};


/* RFC 2960.  Section 3.3.5 Heartbeat Request (HEARTBEAT) (4):
 *
 *  An endpoint should send this chunk to its peer endpoint to probe the
 *  reachability of a particular destination transport address defined in
 *  the present association.
 */

struct sctp_heartbeathdr {
        struct sctp_paramhdr info;
};

struct sctp_heartbeat_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_heartbeathdr hb_hdr;
};


/* PAD chunk could be bundled with heartbeat chunk to probe pmtu */
struct sctp_pad_chunk {
        struct sctp_chunkhdr uh;
};


/* For the abort and shutdown ACK we must carry the init tag in the
 * common header. Just the common header is all that is needed with a
 * chunk descriptor.
 */
struct sctp_abort_chunk {
        struct sctp_chunkhdr uh;
};


/* For the graceful shutdown we must carry the tag (in common header)
 * and the highest consecutive acking value.
 */
struct sctp_shutdownhdr {
        __be32 cum_tsn_ack;
};

struct sctp_shutdown_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_shutdownhdr shutdown_hdr;
};

/* RFC 2960.  Section 3.3.10 Operation Error (ERROR) (9) */

struct sctp_errhdr {
        __be16 cause;
        __be16 length;
        /* __u8  variable[]; */
};

struct sctp_operr_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_errhdr err_hdr;
};

/* RFC 2960 3.3.10 - Operation Error
 *
 * Cause Code: 16 bits (unsigned integer)
 *
 *     Defines the type of error conditions being reported.
 *    Cause Code
 *     Value           Cause Code
 *     ---------      ----------------
 *      1              Invalid Stream Identifier
 *      2              Missing Mandatory Parameter
 *      3              Stale Cookie Error
 *      4              Out of Resource
 *      5              Unresolvable Address
 *      6              Unrecognized Chunk Type
 *      7              Invalid Mandatory Parameter
 *      8              Unrecognized Parameters
 *      9              No User Data
 *     10              Cookie Received While Shutting Down
 */
enum sctp_error {

        SCTP_ERROR_NO_ERROR           = cpu_to_be16(0x00),
        SCTP_ERROR_INV_STRM           = cpu_to_be16(0x01),
        SCTP_ERROR_MISS_PARAM            = cpu_to_be16(0x02),
        SCTP_ERROR_STALE_COOKIE           = cpu_to_be16(0x03),
        SCTP_ERROR_NO_RESOURCE            = cpu_to_be16(0x04),
        SCTP_ERROR_DNS_FAILED      = cpu_to_be16(0x05),
        SCTP_ERROR_UNKNOWN_CHUNK   = cpu_to_be16(0x06),
        SCTP_ERROR_INV_PARAM       = cpu_to_be16(0x07),
        SCTP_ERROR_UNKNOWN_PARAM   = cpu_to_be16(0x08),
        SCTP_ERROR_NO_DATA         = cpu_to_be16(0x09),
        SCTP_ERROR_COOKIE_IN_SHUTDOWN = cpu_to_be16(0x0a),


        /* SCTP Implementation Guide:
         *  11  Restart of an association with new addresses
         *  12  User Initiated Abort
         *  13  Protocol Violation
         *  14  Restart of an Association with New Encapsulation Port
         */

        SCTP_ERROR_RESTART         = cpu_to_be16(0x0b),
        SCTP_ERROR_USER_ABORT      = cpu_to_be16(0x0c),
        SCTP_ERROR_PROTO_VIOLATION = cpu_to_be16(0x0d),
        SCTP_ERROR_NEW_ENCAP_PORT  = cpu_to_be16(0x0e),

        /* ADDIP Section 3.3  New Error Causes
         *
         * Four new Error Causes are added to the SCTP Operational Errors,
         * primarily for use in the ASCONF-ACK chunk.
         *
         * Value          Cause Code
         * ---------      ----------------
         * 0x00A0          Request to Delete Last Remaining IP Address.
         * 0x00A1          Operation Refused Due to Resource Shortage.
         * 0x00A2          Request to Delete Source IP Address.
         * 0x00A3          Association Aborted due to illegal ASCONF-ACK
         * 0x00A4          Request refused - no authorization.
         */
        SCTP_ERROR_DEL_LAST_IP        = cpu_to_be16(0x00A0),
        SCTP_ERROR_RSRC_LOW        = cpu_to_be16(0x00A1),
        SCTP_ERROR_DEL_SRC_IP        = cpu_to_be16(0x00A2),
        SCTP_ERROR_ASCONF_ACK   = cpu_to_be16(0x00A3),
        SCTP_ERROR_REQ_REFUSED        = cpu_to_be16(0x00A4),

        /* AUTH Section 4.  New Error Cause
         *
         * This section defines a new error cause that will be sent if an AUTH
         * chunk is received with an unsupported HMAC identifier.
         * illustrates the new error cause.
         *
         * Cause Code      Error Cause Name
         * --------------------------------------------------------------
         * 0x0105          Unsupported HMAC Identifier
         */
         SCTP_ERROR_UNSUP_HMAC        = cpu_to_be16(0x0105)
};



/* RFC 2960.  Appendix A.  Explicit Congestion Notification.
 *   Explicit Congestion Notification Echo (ECNE) (12)
 */
struct sctp_ecnehdr {
        __be32 lowest_tsn;
};

struct sctp_ecne_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_ecnehdr ence_hdr;
};

/* RFC 2960.  Appendix A.  Explicit Congestion Notification.
 *   Congestion Window Reduced (CWR) (13)
 */
struct sctp_cwrhdr {
        __be32 lowest_tsn;
};

/* PR-SCTP
 * 3.2 Forward Cumulative TSN Chunk Definition (FORWARD TSN)
 *
 * Forward Cumulative TSN chunk has the following format:
 *
 *        0                   1                   2                   3
 *        0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 *      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *      |   Type = 192  |  Flags = 0x00 |        Length = Variable      |
 *      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *      |                      New Cumulative TSN                       |
 *      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *      |         Stream-1              |       Stream Sequence-1       |
 *      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *      \                                                               /
 *      /                                                               \
 *      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *      |         Stream-N              |       Stream Sequence-N       |
 *      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *      Chunk Flags:
 *
 *        Set to all zeros on transmit and ignored on receipt.
 *
 *      New Cumulative TSN: 32 bit u_int
 *
 *       This indicates the new cumulative TSN to the data receiver. Upon
 *       the reception of this value, the data receiver MUST consider
 *       any missing TSNs earlier than or equal to this value as received
 *       and stop reporting them as gaps in any subsequent SACKs.
 *
 *      Stream-N: 16 bit u_int
 *
 *       This field holds a stream number that was skipped by this
 *       FWD-TSN.
 *
 *      Stream Sequence-N: 16 bit u_int
 *       This field holds the sequence number associated with the stream
 *       that was skipped. The stream sequence field holds the largest stream
 *       sequence number in this stream being skipped.  The receiver of
 *       the FWD-TSN's can use the Stream-N and Stream Sequence-N fields
 *       to enable delivery of any stranded TSN's that remain on the stream
 *       re-ordering queues. This field MUST NOT report TSN's corresponding
 *       to DATA chunk that are marked as unordered. For ordered DATA
 *       chunks this field MUST be filled in.
 */
struct sctp_fwdtsn_skip {
        __be16 stream;
        __be16 ssn;
};

struct sctp_fwdtsn_hdr {
        __be32 new_cum_tsn;
        /* struct sctp_fwdtsn_skip skip[]; */
};

struct sctp_fwdtsn_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_fwdtsn_hdr fwdtsn_hdr;
};

struct sctp_ifwdtsn_skip {
        __be16 stream;
        __u8 reserved;
        __u8 flags;
        __be32 mid;
};

struct sctp_ifwdtsn_hdr {
        __be32 new_cum_tsn;
        /* struct sctp_ifwdtsn_skip skip[]; */
};

struct sctp_ifwdtsn_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_ifwdtsn_hdr fwdtsn_hdr;
};

/* ADDIP
 * Section 3.1.1 Address Configuration Change Chunk (ASCONF)
 *
 *         Serial Number: 32 bits (unsigned integer)
 *        This value represents a Serial Number for the ASCONF Chunk. The
 *        valid range of Serial Number is from 0 to 2^32-1.
 *        Serial Numbers wrap back to 0 after reaching 2^32 -1.
 *
 *        Address Parameter: 8 or 20 bytes (depending on type)
 *        The address is an address of the sender of the ASCONF chunk,
 *        the address MUST be considered part of the association by the
 *        peer endpoint. This field may be used by the receiver of the 
 *        ASCONF to help in finding the association. This parameter MUST
 *        be present in every ASCONF message i.e. it is a mandatory TLV
 *        parameter.
 *
 *        ASCONF Parameter: TLV format
 *        Each Address configuration change is represented by a TLV
 *        parameter as defined in Section 3.2. One or more requests may
 *        be present in an ASCONF Chunk.
 *
 * Section 3.1.2 Address Configuration Acknowledgement Chunk (ASCONF-ACK)
 * 
 *        Serial Number: 32 bits (unsigned integer)
 *        This value represents the Serial Number for the received ASCONF
 *        Chunk that is acknowledged by this chunk. This value is copied
 *        from the received ASCONF Chunk. 
 *
 *        ASCONF Parameter Response: TLV format
 *        The ASCONF Parameter Response is used in the ASCONF-ACK to
 *        report status of ASCONF processing.
 */
struct sctp_addip_param {
        struct sctp_paramhdr param_hdr;
        __be32 crr_id;
};

struct sctp_addiphdr {
        __be32        serial;
        /* __u8        params[]; */
};

struct sctp_addip_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_addiphdr addip_hdr;
};

/* AUTH
 * Section 4.1  Authentication Chunk (AUTH)
 *
 *   This chunk is used to hold the result of the HMAC calculation.
 *
 *    0                   1                   2                   3
 *    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *   | Type = 0x0F   |   Flags=0     |             Length            |
 *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *   |     Shared Key Identifier     |   HMAC Identifier             |
 *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *   |                                                               |
 *   \                             HMAC                              /
 *   /                                                               \
 *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *   Type: 1 byte (unsigned integer)
 *           This value MUST be set to 0x0F for  all AUTH-chunks.
 *
 *   Flags: 1 byte (unsigned integer)
 *        Set to zero on transmit and ignored on receipt.
 *
 *   Length: 2 bytes (unsigned integer)
 *           This value holds the length of the HMAC in bytes plus 8.
 *
 *  Shared Key Identifier: 2 bytes (unsigned integer)
 *        This value describes which endpoint pair shared key is used.
 *
 *   HMAC Identifier: 2 bytes (unsigned integer)
 *           This value describes which message digest is being used.  Table 2
 *        shows the currently defined values.
 *
 *    The following Table 2 shows the currently defined values for HMAC
 *       identifiers.
 *
 *         +-----------------+--------------------------+
 *         | HMAC Identifier | Message Digest Algorithm |
 *         +-----------------+--------------------------+
 *         | 0               | Reserved                 |
 *         | 1               | SHA-1 defined in [8]     |
 *         | 2               | Reserved                 |
 *         | 3               | SHA-256 defined in [8]   |
 *         +-----------------+--------------------------+
 *
 *
 *   HMAC: n bytes (unsigned integer) This hold the result of the HMAC
 *      calculation.
 */
struct sctp_authhdr {
        __be16 shkey_id;
        __be16 hmac_id;
        /* __u8   hmac[]; */
};

struct sctp_auth_chunk {
        struct sctp_chunkhdr chunk_hdr;
        struct sctp_authhdr auth_hdr;
};

struct sctp_infox {
        struct sctp_info *sctpinfo;
        struct sctp_association *asoc;
};

struct sctp_reconf_chunk {
        struct sctp_chunkhdr chunk_hdr;
        /* __u8 params[]; */
};

struct sctp_strreset_outreq {
        struct sctp_paramhdr param_hdr;
        __be32 request_seq;
        __be32 response_seq;
        __be32 send_reset_at_tsn;
        __be16 list_of_streams[];
};

struct sctp_strreset_inreq {
        struct sctp_paramhdr param_hdr;
        __be32 request_seq;
        __be16 list_of_streams[];
};

struct sctp_strreset_tsnreq {
        struct sctp_paramhdr param_hdr;
        __be32 request_seq;
};

struct sctp_strreset_addstrm {
        struct sctp_paramhdr param_hdr;
        __be32 request_seq;
        __be16 number_of_streams;
        __be16 reserved;
};

enum {
        SCTP_STRRESET_NOTHING_TO_DO        = 0x00,
        SCTP_STRRESET_PERFORMED                = 0x01,
        SCTP_STRRESET_DENIED                = 0x02,
        SCTP_STRRESET_ERR_WRONG_SSN        = 0x03,
        SCTP_STRRESET_ERR_IN_PROGRESS        = 0x04,
        SCTP_STRRESET_ERR_BAD_SEQNO        = 0x05,
        SCTP_STRRESET_IN_PROGRESS        = 0x06,
};

struct sctp_strreset_resp {
        struct sctp_paramhdr param_hdr;
        __be32 response_seq;
        __be32 result;
};

struct sctp_strreset_resptsn {
        struct sctp_paramhdr param_hdr;
        __be32 response_seq;
        __be32 result;
        __be32 senders_next_tsn;
        __be32 receivers_next_tsn;
};

enum {
        SCTP_DSCP_SET_MASK = 0x1,
        SCTP_DSCP_VAL_MASK = 0xfc,
        SCTP_FLOWLABEL_SET_MASK = 0x100000,
        SCTP_FLOWLABEL_VAL_MASK = 0xfffff
};

/* UDP Encapsulation
 * draft-tuexen-tsvwg-sctp-udp-encaps-cons-03.html#section-4-4
 *
 *   The error cause indicating an "Restart of an Association with
 *   New Encapsulation Port"
 *
 * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |        Cause Code = 14        |       Cause Length = 8        |
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |   Current Encapsulation Port  |     New Encapsulation Port    |
 * +-------------------------------+-------------------------------+
 */
struct sctp_new_encap_port_hdr {
        __be16 cur_port;
        __be16 new_port;
};

/* Round an int up to the next multiple of 4.  */
#define SCTP_PAD4(s) (((s)+3)&~3)
/* Truncate to the previous multiple of 4.  */
#define SCTP_TRUNC4(s) ((s)&~3)

#endif /* __LINUX_SCTP_H__ */



























   98 
   98 

   98 














  116 


    3 

  117 




  117 







  117 



  117 







  117 



  117 




























  115 



  114 








  115 

  115 













    3 
















    3 




    3 





































  416 
















































  116 
  117 
  117 

















  117 








  116 

  117 

  117 










  117 
    4 

  113 


  117 


  114 






  115 


  115 





  114 
  106 


   11 


   11 







  117 


























  115 















  115 


  115 






  114 
  114 
  115 
  115 

  115 
   98 



















































  117 




  117 

  113 





    4 
  112 






  116 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2008 IBM Corporation
 *
 * Author: Mimi Zohar <zohar@us.ibm.com>
 *
 * File: ima_api.c
 *        Implements must_appraise_or_measure, collect_measurement,
 *        appraise_measurement, store_measurement and store_template.
 */
#include <linux/slab.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/xattr.h>
#include <linux/evm.h>
#include <linux/fsverity.h>

#include "ima.h"

/*
 * ima_free_template_entry - free an existing template entry
 */
void ima_free_template_entry(struct ima_template_entry *entry)
{
        int i;

        for (i = 0; i < entry->template_desc->num_fields; i++)
                kfree(entry->template_data[i].data);

        kfree(entry->digests);
        kfree(entry);
}

/*
 * ima_alloc_init_template - create and initialize a new template entry
 */
int ima_alloc_init_template(struct ima_event_data *event_data,
                            struct ima_template_entry **entry,
                            struct ima_template_desc *desc)
{
        struct ima_template_desc *template_desc;
        struct tpm_digest *digests;
        int i, result = 0;

        if (desc)
                template_desc = desc;
        else
                template_desc = ima_template_desc_current();

        *entry = kzalloc(struct_size(*entry, template_data,
                                     template_desc->num_fields), GFP_NOFS);
        if (!*entry)
                return -ENOMEM;

        digests = kcalloc(NR_BANKS(ima_tpm_chip) + ima_extra_slots,
                          sizeof(*digests), GFP_NOFS);
        if (!digests) {
                kfree(*entry);
                *entry = NULL;
                return -ENOMEM;
        }

        (*entry)->digests = digests;
        (*entry)->template_desc = template_desc;
        for (i = 0; i < template_desc->num_fields; i++) {
                const struct ima_template_field *field =
                        template_desc->fields[i];
                u32 len;

                result = field->field_init(event_data,
                                           &((*entry)->template_data[i]));
                if (result != 0)
                        goto out;

                len = (*entry)->template_data[i].len;
                (*entry)->template_data_len += sizeof(len);
                (*entry)->template_data_len += len;
        }
        return 0;
out:
        ima_free_template_entry(*entry);
        *entry = NULL;
        return result;
}

/*
 * ima_store_template - store ima template measurements
 *
 * Calculate the hash of a template entry, add the template entry
 * to an ordered list of measurement entries maintained inside the kernel,
 * and also update the aggregate integrity value (maintained inside the
 * configured TPM PCR) over the hashes of the current list of measurement
 * entries.
 *
 * Applications retrieve the current kernel-held measurement list through
 * the securityfs entries in /sys/kernel/security/ima. The signed aggregate
 * TPM PCR (called quote) can be retrieved using a TPM user space library
 * and is used to validate the measurement list.
 *
 * Returns 0 on success, error code otherwise
 */
int ima_store_template(struct ima_template_entry *entry,
                       int violation, struct inode *inode,
                       const unsigned char *filename, int pcr)
{
        static const char op[] = "add_template_measure";
        static const char audit_cause[] = "hashing_error";
        char *template_name = entry->template_desc->name;
        int result;

        if (!violation) {
                result = ima_calc_field_array_hash(&entry->template_data[0],
                                                   entry);
                if (result < 0) {
                        integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode,
                                            template_name, op,
                                            audit_cause, result, 0);
                        return result;
                }
        }
        entry->pcr = pcr;
        result = ima_add_template_entry(entry, violation, op, inode, filename);
        return result;
}

/*
 * ima_add_violation - add violation to measurement list.
 *
 * Violations are flagged in the measurement list with zero hash values.
 * By extending the PCR with 0xFF's instead of with zeroes, the PCR
 * value is invalidated.
 */
void ima_add_violation(struct file *file, const unsigned char *filename,
                       struct ima_iint_cache *iint, const char *op,
                       const char *cause)
{
        struct ima_template_entry *entry;
        struct inode *inode = file_inode(file);
        struct ima_event_data event_data = { .iint = iint,
                                             .file = file,
                                             .filename = filename,
                                             .violation = cause };
        int violation = 1;
        int result;

        /* can overflow, only indicator */
        atomic_long_inc(&ima_htable.violations);

        result = ima_alloc_init_template(&event_data, &entry, NULL);
        if (result < 0) {
                result = -ENOMEM;
                goto err_out;
        }
        result = ima_store_template(entry, violation, inode,
                                    filename, CONFIG_IMA_MEASURE_PCR_IDX);
        if (result < 0)
                ima_free_template_entry(entry);
err_out:
        integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode, filename,
                            op, cause, result, 0);
}

/**
 * ima_get_action - appraise & measure decision based on policy.
 * @idmap: idmap of the mount the inode was found from
 * @inode: pointer to the inode associated with the object being validated
 * @cred: pointer to credentials structure to validate
 * @prop: properties of the task being validated
 * @mask: contains the permission mask (MAY_READ, MAY_WRITE, MAY_EXEC,
 *        MAY_APPEND)
 * @func: caller identifier
 * @pcr: pointer filled in if matched measure policy sets pcr=
 * @template_desc: pointer filled in if matched measure policy sets template=
 * @func_data: func specific data, may be NULL
 * @allowed_algos: allowlist of hash algorithms for the IMA xattr
 *
 * The policy is defined in terms of keypairs:
 *                subj=, obj=, type=, func=, mask=, fsmagic=
 *        subj,obj, and type: are LSM specific.
 *        func: FILE_CHECK | BPRM_CHECK | CREDS_CHECK | MMAP_CHECK | MODULE_CHECK
 *        | KEXEC_CMDLINE | KEY_CHECK | CRITICAL_DATA | SETXATTR_CHECK
 *        | MMAP_CHECK_REQPROT
 *        mask: contains the permission mask
 *        fsmagic: hex value
 *
 * Returns IMA_MEASURE, IMA_APPRAISE mask.
 *
 */
int ima_get_action(struct mnt_idmap *idmap, struct inode *inode,
                   const struct cred *cred, struct lsm_prop *prop, int mask,
                   enum ima_hooks func, int *pcr,
                   struct ima_template_desc **template_desc,
                   const char *func_data, unsigned int *allowed_algos)
{
        int flags = IMA_MEASURE | IMA_AUDIT | IMA_APPRAISE | IMA_HASH;

        flags &= ima_policy_flag;

        return ima_match_policy(idmap, inode, cred, prop, func, mask,
                                flags, pcr, template_desc, func_data,
                                allowed_algos);
}

static bool ima_get_verity_digest(struct ima_iint_cache *iint,
                                  struct inode *inode,
                                  struct ima_max_digest_data *hash)
{
        enum hash_algo alg;
        int digest_len;

        /*
         * On failure, 'measure' policy rules will result in a file data
         * hash containing 0's.
         */
        digest_len = fsverity_get_digest(inode, hash->digest, NULL, &alg);
        if (digest_len == 0)
                return false;

        /*
         * Unlike in the case of actually calculating the file hash, in
         * the fsverity case regardless of the hash algorithm, return
         * the verity digest to be included in the measurement list. A
         * mismatch between the verity algorithm and the xattr signature
         * algorithm, if one exists, will be detected later.
         */
        hash->hdr.algo = alg;
        hash->hdr.length = digest_len;
        return true;
}

/*
 * ima_collect_measurement - collect file measurement
 *
 * Calculate the file hash, if it doesn't already exist,
 * storing the measurement and i_version in the iint.
 *
 * Must be called with iint->mutex held.
 *
 * Return 0 on success, error code otherwise
 */
int ima_collect_measurement(struct ima_iint_cache *iint, struct file *file,
                            void *buf, loff_t size, enum hash_algo algo,
                            struct modsig *modsig)
{
        const char *audit_cause = "failed";
        struct inode *inode = file_inode(file);
        struct inode *real_inode = d_real_inode(file_dentry(file));
        struct ima_max_digest_data hash;
        struct ima_digest_data *hash_hdr = container_of(&hash.hdr,
                                                struct ima_digest_data, hdr);
        struct name_snapshot filename;
        struct kstat stat;
        int result = 0;
        int length;
        void *tmpbuf;
        u64 i_version = 0;

        /*
         * Always collect the modsig, because IMA might have already collected
         * the file digest without collecting the modsig in a previous
         * measurement rule.
         */
        if (modsig)
                ima_collect_modsig(modsig, buf, size);

        if (iint->flags & IMA_COLLECTED)
                goto out;

        /*
         * Detecting file change is based on i_version. On filesystems
         * which do not support i_version, support was originally limited
         * to an initial measurement/appraisal/audit, but was modified to
         * assume the file changed.
         */
        result = vfs_getattr_nosec(&file->f_path, &stat, STATX_CHANGE_COOKIE,
                                   AT_STATX_SYNC_AS_STAT);
        if (!result && (stat.result_mask & STATX_CHANGE_COOKIE))
                i_version = stat.change_cookie;
        hash.hdr.algo = algo;
        hash.hdr.length = hash_digest_size[algo];

        /* Initialize hash digest to 0's in case of failure */
        memset(&hash.digest, 0, sizeof(hash.digest));

        if (iint->flags & IMA_VERITY_REQUIRED) {
                if (!ima_get_verity_digest(iint, inode, &hash)) {
                        audit_cause = "no-verity-digest";
                        result = -ENODATA;
                }
        } else if (buf) {
                result = ima_calc_buffer_hash(buf, size, hash_hdr);
        } else {
                result = ima_calc_file_hash(file, hash_hdr);
        }

        if (result && result != -EBADF && result != -EINVAL)
                goto out;

        length = sizeof(hash.hdr) + hash.hdr.length;
        tmpbuf = krealloc(iint->ima_hash, length, GFP_NOFS);
        if (!tmpbuf) {
                result = -ENOMEM;
                goto out;
        }

        iint->ima_hash = tmpbuf;
        memcpy(iint->ima_hash, &hash, length);
        if (real_inode == inode)
                iint->real_inode.version = i_version;
        else
                integrity_inode_attrs_store(&iint->real_inode, i_version,
                                            real_inode);

        /* Possibly temporary failure due to type of read (eg. O_DIRECT) */
        if (!result)
                iint->flags |= IMA_COLLECTED;
out:
        if (result) {
                if (file->f_flags & O_DIRECT)
                        audit_cause = "failed(directio)";

                take_dentry_name_snapshot(&filename, file->f_path.dentry);

                integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode,
                                    filename.name.name, "collect_data",
                                    audit_cause, result, 0);

                release_dentry_name_snapshot(&filename);
        }
        return result;
}

/*
 * ima_store_measurement - store file measurement
 *
 * Create an "ima" template and then store the template by calling
 * ima_store_template.
 *
 * We only get here if the inode has not already been measured,
 * but the measurement could already exist:
 *        - multiple copies of the same file on either the same or
 *          different filesystems.
 *        - the inode was previously flushed as well as the iint info,
 *          containing the hashing info.
 *
 * Must be called with iint->mutex held.
 */
void ima_store_measurement(struct ima_iint_cache *iint, struct file *file,
                           const unsigned char *filename,
                           struct evm_ima_xattr_data *xattr_value,
                           int xattr_len, const struct modsig *modsig, int pcr,
                           struct ima_template_desc *template_desc)
{
        static const char op[] = "add_template_measure";
        static const char audit_cause[] = "ENOMEM";
        int result = -ENOMEM;
        struct inode *inode = file_inode(file);
        struct ima_template_entry *entry;
        struct ima_event_data event_data = { .iint = iint,
                                             .file = file,
                                             .filename = filename,
                                             .xattr_value = xattr_value,
                                             .xattr_len = xattr_len,
                                             .modsig = modsig };
        int violation = 0;

        /*
         * We still need to store the measurement in the case of MODSIG because
         * we only have its contents to put in the list at the time of
         * appraisal, but a file measurement from earlier might already exist in
         * the measurement list.
         */
        if (iint->measured_pcrs & (0x1 << pcr) && !modsig)
                return;

        result = ima_alloc_init_template(&event_data, &entry, template_desc);
        if (result < 0) {
                integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode, filename,
                                    op, audit_cause, result, 0);
                return;
        }

        result = ima_store_template(entry, violation, inode, filename, pcr);
        if ((!result || result == -EEXIST) && !(file->f_flags & O_DIRECT)) {
                iint->flags |= IMA_MEASURED;
                iint->measured_pcrs |= (0x1 << pcr);
        }
        if (result < 0)
                ima_free_template_entry(entry);
}

void ima_audit_measurement(struct ima_iint_cache *iint,
                           const unsigned char *filename)
{
        struct audit_buffer *ab;
        char *hash;
        const char *algo_name = hash_algo_name[iint->ima_hash->algo];
        int i;

        if (iint->flags & IMA_AUDITED)
                return;

        hash = kzalloc((iint->ima_hash->length * 2) + 1, GFP_KERNEL);
        if (!hash)
                return;

        for (i = 0; i < iint->ima_hash->length; i++)
                hex_byte_pack(hash + (i * 2), iint->ima_hash->digest[i]);
        hash[i * 2] = '\0';

        ab = audit_log_start(audit_context(), GFP_KERNEL,
                             AUDIT_INTEGRITY_RULE);
        if (!ab)
                goto out;

        audit_log_format(ab, "file=");
        audit_log_untrustedstring(ab, filename);
        audit_log_format(ab, " hash=\"%s:%s\"", algo_name, hash);

        audit_log_task_info(ab);
        audit_log_end(ab);

        iint->flags |= IMA_AUDITED;
out:
        kfree(hash);
        return;
}

/*
 * ima_d_path - return a pointer to the full pathname
 *
 * Attempt to return a pointer to the full pathname for use in the
 * IMA measurement list, IMA audit records, and auditing logs.
 *
 * On failure, return a pointer to a copy of the filename, not dname.
 * Returning a pointer to dname, could result in using the pointer
 * after the memory has been freed.
 */
const char *ima_d_path(const struct path *path, char **pathbuf, char *namebuf)
{
        struct name_snapshot filename;
        char *pathname = NULL;

        *pathbuf = __getname();
        if (*pathbuf) {
                pathname = d_absolute_path(path, *pathbuf, PATH_MAX);
                if (IS_ERR(pathname)) {
                        __putname(*pathbuf);
                        *pathbuf = NULL;
                        pathname = NULL;
                }
        }

        if (!pathname) {
                take_dentry_name_snapshot(&filename, path->dentry);
                strscpy(namebuf, filename.name.name, NAME_MAX);
                release_dentry_name_snapshot(&filename);

                pathname = namebuf;
        }

        return pathname;
}






































   20 



   58 
   20 



   18 
   20 




























































   54 


   54 


   52 
   53 








   53 

















   54 

   54 
   53 




















   85 
















   86 


















   87 

   87 
   87 
   86 









   86 




   86 


   85 


   88 














































   55 


   57 

































   57 


   58 
   57 
   53 




















































































































































    1 


    1 

    1 












    4 








    3 
    1 











    3 




    4 














   53 


   50 






   52 































   45 








   50 








   46 



   49 
   48 



   14 




   47 








   46 





   44 
   45 

    2 




   47 

   47 







   47 




   16 









   57 




   55 
   49 















   57 
















   57 
   57 




   57 
   55 
    1 
    1 









   55 




   56 






   57 










   36 


   25 



   19 














   20 




















   20 
   20 







   20 


   20 










   20 
    6 

















   20 


























   35 





   20 








   45 

   46 






   56 










































   58 






   55 


   58 
   56 

    4 


   54 

   47 
   56 

















   46 













   48 






   48 
   48 
   48 


   48 







































































































































    3 














    3 





    3 

    3 















    3 










    3 
    3 


    3 


    3 























    3 
    3 






    1 








    1 
    1 











   20 











   19 
   19 
   20 
   20 










   11 


   20 





   20 

   11 

















































   19 
   19 




   19 





   20 











   19 

   20 




   20 

   20 
















   19 


   19 














   19 
   19 


   19 













   20 


   20 
    1 

    1 

    1 

   20 


   19 







   20 



























   20 







   20 








   20 

   20 

   19 


   20 


   20 









    3 


    3 


















    3 












    1 
    1 














    3 



























    3 



    3 

    1 
    3 
    3 


























































































    2 


    2 





    2 
    1 
    2 




    2 
    2 












    2 

    3 
    2 
    3 












































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
// SPDX-License-Identifier: GPL-2.0-or-later
/* Keyring handling
 *
 * Copyright (C) 2004-2005, 2008, 2013 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/export.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/security.h>
#include <linux/seq_file.h>
#include <linux/err.h>
#include <linux/user_namespace.h>
#include <linux/nsproxy.h>
#include <keys/keyring-type.h>
#include <keys/user-type.h>
#include <linux/assoc_array_priv.h>
#include <linux/uaccess.h>
#include <net/net_namespace.h>
#include "internal.h"

/*
 * When plumbing the depths of the key tree, this sets a hard limit
 * set on how deep we're willing to go.
 */
#define KEYRING_SEARCH_MAX_DEPTH 6

/*
 * We mark pointers we pass to the associative array with bit 1 set if
 * they're keyrings and clear otherwise.
 */
#define KEYRING_PTR_SUBTYPE        0x2UL

static inline bool keyring_ptr_is_keyring(const struct assoc_array_ptr *x)
{
        return (unsigned long)x & KEYRING_PTR_SUBTYPE;
}
static inline struct key *keyring_ptr_to_key(const struct assoc_array_ptr *x)
{
        void *object = assoc_array_ptr_to_leaf(x);
        return (struct key *)((unsigned long)object & ~KEYRING_PTR_SUBTYPE);
}
static inline void *keyring_key_to_ptr(struct key *key)
{
        if (key->type == &key_type_keyring)
                return (void *)((unsigned long)key | KEYRING_PTR_SUBTYPE);
        return key;
}

static DEFINE_RWLOCK(keyring_name_lock);

/*
 * Clean up the bits of user_namespace that belong to us.
 */
void key_free_user_ns(struct user_namespace *ns)
{
        write_lock(&keyring_name_lock);
        list_del_init(&ns->keyring_name_list);
        write_unlock(&keyring_name_lock);

        key_put(ns->user_keyring_register);
#ifdef CONFIG_PERSISTENT_KEYRINGS
        key_put(ns->persistent_keyring_register);
#endif
}

/*
 * The keyring key type definition.  Keyrings are simply keys of this type and
 * can be treated as ordinary keys in addition to having their own special
 * operations.
 */
static int keyring_preparse(struct key_preparsed_payload *prep);
static void keyring_free_preparse(struct key_preparsed_payload *prep);
static int keyring_instantiate(struct key *keyring,
                               struct key_preparsed_payload *prep);
static void keyring_revoke(struct key *keyring);
static void keyring_destroy(struct key *keyring);
static void keyring_describe(const struct key *keyring, struct seq_file *m);
static long keyring_read(const struct key *keyring,
                         char *buffer, size_t buflen);

struct key_type key_type_keyring = {
        .name                = "keyring",
        .def_datalen        = 0,
        .preparse        = keyring_preparse,
        .free_preparse        = keyring_free_preparse,
        .instantiate        = keyring_instantiate,
        .revoke                = keyring_revoke,
        .destroy        = keyring_destroy,
        .describe        = keyring_describe,
        .read                = keyring_read,
};
EXPORT_SYMBOL(key_type_keyring);

/*
 * Semaphore to serialise link/link calls to prevent two link calls in parallel
 * introducing a cycle.
 */
static DEFINE_MUTEX(keyring_serialise_link_lock);

/*
 * Publish the name of a keyring so that it can be found by name (if it has
 * one and it doesn't begin with a dot).
 */
static void keyring_publish_name(struct key *keyring)
{
        struct user_namespace *ns = current_user_ns();

        if (keyring->description &&
            keyring->description[0] &&
            keyring->description[0] != '.') {
                write_lock(&keyring_name_lock);
                list_add_tail(&keyring->name_link, &ns->keyring_name_list);
                write_unlock(&keyring_name_lock);
        }
}

/*
 * Preparse a keyring payload
 */
static int keyring_preparse(struct key_preparsed_payload *prep)
{
        return prep->datalen != 0 ? -EINVAL : 0;
}

/*
 * Free a preparse of a user defined key payload
 */
static void keyring_free_preparse(struct key_preparsed_payload *prep)
{
}

/*
 * Initialise a keyring.
 *
 * Returns 0 on success, -EINVAL if given any data.
 */
static int keyring_instantiate(struct key *keyring,
                               struct key_preparsed_payload *prep)
{
        assoc_array_init(&keyring->keys);
        /* make the keyring available by name if it has one */
        keyring_publish_name(keyring);
        return 0;
}

/*
 * Multiply 64-bits by 32-bits to 96-bits and fold back to 64-bit.  Ideally we'd
 * fold the carry back too, but that requires inline asm.
 */
static u64 mult_64x32_and_fold(u64 x, u32 y)
{
        u64 hi = (u64)(u32)(x >> 32) * y;
        u64 lo = (u64)(u32)(x) * y;
        return lo + ((u64)(u32)hi << 32) + (u32)(hi >> 32);
}

/*
 * Hash a key type and description.
 */
static void hash_key_type_and_desc(struct keyring_index_key *index_key)
{
        const unsigned level_shift = ASSOC_ARRAY_LEVEL_STEP;
        const unsigned long fan_mask = ASSOC_ARRAY_FAN_MASK;
        const char *description = index_key->description;
        unsigned long hash, type;
        u32 piece;
        u64 acc;
        int n, desc_len = index_key->desc_len;

        type = (unsigned long)index_key->type;
        acc = mult_64x32_and_fold(type, desc_len + 13);
        acc = mult_64x32_and_fold(acc, 9207);
        piece = (unsigned long)index_key->domain_tag;
        acc = mult_64x32_and_fold(acc, piece);
        acc = mult_64x32_and_fold(acc, 9207);

        for (;;) {
                n = desc_len;
                if (n <= 0)
                        break;
                if (n > 4)
                        n = 4;
                piece = 0;
                memcpy(&piece, description, n);
                description += n;
                desc_len -= n;
                acc = mult_64x32_and_fold(acc, piece);
                acc = mult_64x32_and_fold(acc, 9207);
        }

        /* Fold the hash down to 32 bits if need be. */
        hash = acc;
        if (ASSOC_ARRAY_KEY_CHUNK_SIZE == 32)
                hash ^= acc >> 32;

        /* Squidge all the keyrings into a separate part of the tree to
         * ordinary keys by making sure the lowest level segment in the hash is
         * zero for keyrings and non-zero otherwise.
         */
        if (index_key->type != &key_type_keyring && (hash & fan_mask) == 0)
                hash |= (hash >> (ASSOC_ARRAY_KEY_CHUNK_SIZE - level_shift)) | 1;
        else if (index_key->type == &key_type_keyring && (hash & fan_mask) != 0)
                hash = (hash + (hash << level_shift)) & ~fan_mask;
        index_key->hash = hash;
}

/*
 * Finalise an index key to include a part of the description actually in the
 * index key, to set the domain tag and to calculate the hash.
 */
void key_set_index_key(struct keyring_index_key *index_key)
{
        static struct key_tag default_domain_tag = { .usage = REFCOUNT_INIT(1), };
        size_t n = min_t(size_t, index_key->desc_len, sizeof(index_key->desc));

        memcpy(index_key->desc, index_key->description, n);

        if (!index_key->domain_tag) {
                if (index_key->type->flags & KEY_TYPE_NET_DOMAIN)
                        index_key->domain_tag = current->nsproxy->net_ns->key_domain;
                else
                        index_key->domain_tag = &default_domain_tag;
        }

        hash_key_type_and_desc(index_key);
}

/**
 * key_put_tag - Release a ref on a tag.
 * @tag: The tag to release.
 *
 * This releases a reference the given tag and returns true if that ref was the
 * last one.
 */
bool key_put_tag(struct key_tag *tag)
{
        if (refcount_dec_and_test(&tag->usage)) {
                kfree_rcu(tag, rcu);
                return true;
        }

        return false;
}

/**
 * key_remove_domain - Kill off a key domain and gc its keys
 * @domain_tag: The domain tag to release.
 *
 * This marks a domain tag as being dead and releases a ref on it.  If that
 * wasn't the last reference, the garbage collector is poked to try and delete
 * all keys that were in the domain.
 */
void key_remove_domain(struct key_tag *domain_tag)
{
        domain_tag->removed = true;
        if (!key_put_tag(domain_tag))
                key_schedule_gc_links();
}

/*
 * Build the next index key chunk.
 *
 * We return it one word-sized chunk at a time.
 */
static unsigned long keyring_get_key_chunk(const void *data, int level)
{
        const struct keyring_index_key *index_key = data;
        unsigned long chunk = 0;
        const u8 *d;
        int desc_len = index_key->desc_len, n = sizeof(chunk);

        level /= ASSOC_ARRAY_KEY_CHUNK_SIZE;
        switch (level) {
        case 0:
                return index_key->hash;
        case 1:
                return index_key->x;
        case 2:
                return (unsigned long)index_key->type;
        case 3:
                return (unsigned long)index_key->domain_tag;
        default:
                level -= 4;
                if (desc_len <= sizeof(index_key->desc))
                        return 0;

                d = index_key->description + sizeof(index_key->desc);
                d += level * sizeof(long);
                desc_len -= sizeof(index_key->desc);
                if (desc_len > n)
                        desc_len = n;
                do {
                        chunk <<= 8;
                        chunk |= *d++;
                } while (--desc_len > 0);
                return chunk;
        }
}

static unsigned long keyring_get_object_key_chunk(const void *object, int level)
{
        const struct key *key = keyring_ptr_to_key(object);
        return keyring_get_key_chunk(&key->index_key, level);
}

static bool keyring_compare_object(const void *object, const void *data)
{
        const struct keyring_index_key *index_key = data;
        const struct key *key = keyring_ptr_to_key(object);

        return key->index_key.type == index_key->type &&
                key->index_key.domain_tag == index_key->domain_tag &&
                key->index_key.desc_len == index_key->desc_len &&
                memcmp(key->index_key.description, index_key->description,
                       index_key->desc_len) == 0;
}

/*
 * Compare the index keys of a pair of objects and determine the bit position
 * at which they differ - if they differ.
 */
static int keyring_diff_objects(const void *object, const void *data)
{
        const struct key *key_a = keyring_ptr_to_key(object);
        const struct keyring_index_key *a = &key_a->index_key;
        const struct keyring_index_key *b = data;
        unsigned long seg_a, seg_b;
        int level, i;

        level = 0;
        seg_a = a->hash;
        seg_b = b->hash;
        if ((seg_a ^ seg_b) != 0)
                goto differ;
        level += ASSOC_ARRAY_KEY_CHUNK_SIZE / 8;

        /* The number of bits contributed by the hash is controlled by a
         * constant in the assoc_array headers.  Everything else thereafter we
         * can deal with as being machine word-size dependent.
         */
        seg_a = a->x;
        seg_b = b->x;
        if ((seg_a ^ seg_b) != 0)
                goto differ;
        level += sizeof(unsigned long);

        /* The next bit may not work on big endian */
        seg_a = (unsigned long)a->type;
        seg_b = (unsigned long)b->type;
        if ((seg_a ^ seg_b) != 0)
                goto differ;
        level += sizeof(unsigned long);

        seg_a = (unsigned long)a->domain_tag;
        seg_b = (unsigned long)b->domain_tag;
        if ((seg_a ^ seg_b) != 0)
                goto differ;
        level += sizeof(unsigned long);

        i = sizeof(a->desc);
        if (a->desc_len <= i)
                goto same;

        for (; i < a->desc_len; i++) {
                seg_a = *(unsigned char *)(a->description + i);
                seg_b = *(unsigned char *)(b->description + i);
                if ((seg_a ^ seg_b) != 0)
                        goto differ_plus_i;
        }

same:
        return -1;

differ_plus_i:
        level += i;
differ:
        i = level * 8 + __ffs(seg_a ^ seg_b);
        return i;
}

/*
 * Free an object after stripping the keyring flag off of the pointer.
 */
static void keyring_free_object(void *object)
{
        key_put(keyring_ptr_to_key(object));
}

/*
 * Operations for keyring management by the index-tree routines.
 */
static const struct assoc_array_ops keyring_assoc_array_ops = {
        .get_key_chunk                = keyring_get_key_chunk,
        .get_object_key_chunk        = keyring_get_object_key_chunk,
        .compare_object                = keyring_compare_object,
        .diff_objects                = keyring_diff_objects,
        .free_object                = keyring_free_object,
};

/*
 * Clean up a keyring when it is destroyed.  Unpublish its name if it had one
 * and dispose of its data.
 *
 * The garbage collector detects the final key_put(), removes the keyring from
 * the serial number tree and then does RCU synchronisation before coming here,
 * so we shouldn't need to worry about code poking around here with the RCU
 * readlock held by this time.
 */
static void keyring_destroy(struct key *keyring)
{
        if (keyring->description) {
                write_lock(&keyring_name_lock);

                if (keyring->name_link.next != NULL &&
                    !list_empty(&keyring->name_link))
                        list_del(&keyring->name_link);

                write_unlock(&keyring_name_lock);
        }

        if (keyring->restrict_link) {
                struct key_restriction *keyres = keyring->restrict_link;

                key_put(keyres->key);
                kfree(keyres);
        }

        assoc_array_destroy(&keyring->keys, &keyring_assoc_array_ops);
}

/*
 * Describe a keyring for /proc.
 */
static void keyring_describe(const struct key *keyring, struct seq_file *m)
{
        if (keyring->description)
                seq_puts(m, keyring->description);
        else
                seq_puts(m, "[anon]");

        if (key_is_positive(keyring)) {
                if (keyring->keys.nr_leaves_on_tree != 0)
                        seq_printf(m, ": %lu", keyring->keys.nr_leaves_on_tree);
                else
                        seq_puts(m, ": empty");
        }
}

struct keyring_read_iterator_context {
        size_t                        buflen;
        size_t                        count;
        key_serial_t                *buffer;
};

static int keyring_read_iterator(const void *object, void *data)
{
        struct keyring_read_iterator_context *ctx = data;
        const struct key *key = keyring_ptr_to_key(object);

        kenter("{%s,%d},,{%zu/%zu}",
               key->type->name, key->serial, ctx->count, ctx->buflen);

        if (ctx->count >= ctx->buflen)
                return 1;

        *ctx->buffer++ = key->serial;
        ctx->count += sizeof(key->serial);
        return 0;
}

/*
 * Read a list of key IDs from the keyring's contents in binary form
 *
 * The keyring's semaphore is read-locked by the caller.  This prevents someone
 * from modifying it under us - which could cause us to read key IDs multiple
 * times.
 */
static long keyring_read(const struct key *keyring,
                         char *buffer, size_t buflen)
{
        struct keyring_read_iterator_context ctx;
        long ret;

        kenter("{%d},,%zu", key_serial(keyring), buflen);

        if (buflen & (sizeof(key_serial_t) - 1))
                return -EINVAL;

        /* Copy as many key IDs as fit into the buffer */
        if (buffer && buflen) {
                ctx.buffer = (key_serial_t *)buffer;
                ctx.buflen = buflen;
                ctx.count = 0;
                ret = assoc_array_iterate(&keyring->keys,
                                          keyring_read_iterator, &ctx);
                if (ret < 0) {
                        kleave(" = %ld [iterate]", ret);
                        return ret;
                }
        }

        /* Return the size of the buffer needed */
        ret = keyring->keys.nr_leaves_on_tree * sizeof(key_serial_t);
        if (ret <= buflen)
                kleave("= %ld [ok]", ret);
        else
                kleave("= %ld [buffer too small]", ret);
        return ret;
}

/*
 * Allocate a keyring and link into the destination keyring.
 */
struct key *keyring_alloc(const char *description, kuid_t uid, kgid_t gid,
                          const struct cred *cred, key_perm_t perm,
                          unsigned long flags,
                          struct key_restriction *restrict_link,
                          struct key *dest)
{
        struct key *keyring;
        int ret;

        keyring = key_alloc(&key_type_keyring, description,
                            uid, gid, cred, perm, flags, restrict_link);
        if (!IS_ERR(keyring)) {
                ret = key_instantiate_and_link(keyring, NULL, 0, dest, NULL);
                if (ret < 0) {
                        key_put(keyring);
                        keyring = ERR_PTR(ret);
                }
        }

        return keyring;
}
EXPORT_SYMBOL(keyring_alloc);

/**
 * restrict_link_reject - Give -EPERM to restrict link
 * @keyring: The keyring being added to.
 * @type: The type of key being added.
 * @payload: The payload of the key intended to be added.
 * @restriction_key: Keys providing additional data for evaluating restriction.
 *
 * Reject the addition of any links to a keyring.  It can be overridden by
 * passing KEY_ALLOC_BYPASS_RESTRICTION to key_instantiate_and_link() when
 * adding a key to a keyring.
 *
 * This is meant to be stored in a key_restriction structure which is passed
 * in the restrict_link parameter to keyring_alloc().
 */
int restrict_link_reject(struct key *keyring,
                         const struct key_type *type,
                         const union key_payload *payload,
                         struct key *restriction_key)
{
        return -EPERM;
}

/*
 * By default, we keys found by getting an exact match on their descriptions.
 */
bool key_default_cmp(const struct key *key,
                     const struct key_match_data *match_data)
{
        return strcmp(key->description, match_data->raw_data) == 0;
}

/*
 * Iteration function to consider each key found.
 */
static int keyring_search_iterator(const void *object, void *iterator_data)
{
        struct keyring_search_context *ctx = iterator_data;
        const struct key *key = keyring_ptr_to_key(object);
        unsigned long kflags = READ_ONCE(key->flags);
        short state = READ_ONCE(key->state);

        kenter("{%d}", key->serial);

        /* ignore keys not of this type */
        if (key->type != ctx->index_key.type) {
                kleave(" = 0 [!type]");
                return 0;
        }

        /* skip invalidated, revoked and expired keys */
        if (ctx->flags & KEYRING_SEARCH_DO_STATE_CHECK) {
                time64_t expiry = READ_ONCE(key->expiry);

                if (kflags & ((1 << KEY_FLAG_INVALIDATED) |
                              (1 << KEY_FLAG_REVOKED))) {
                        ctx->result = ERR_PTR(-EKEYREVOKED);
                        kleave(" = %d [invrev]", ctx->skipped_ret);
                        goto skipped;
                }

                if (expiry && ctx->now >= expiry) {
                        if (!(ctx->flags & KEYRING_SEARCH_SKIP_EXPIRED))
                                ctx->result = ERR_PTR(-EKEYEXPIRED);
                        kleave(" = %d [expire]", ctx->skipped_ret);
                        goto skipped;
                }
        }

        /* keys that don't match */
        if (!ctx->match_data.cmp(key, &ctx->match_data)) {
                kleave(" = 0 [!match]");
                return 0;
        }

        /* key must have search permissions */
        if (!(ctx->flags & KEYRING_SEARCH_NO_CHECK_PERM) &&
            key_task_permission(make_key_ref(key, ctx->possessed),
                                ctx->cred, KEY_NEED_SEARCH) < 0) {
                ctx->result = ERR_PTR(-EACCES);
                kleave(" = %d [!perm]", ctx->skipped_ret);
                goto skipped;
        }

        if (ctx->flags & KEYRING_SEARCH_DO_STATE_CHECK) {
                /* we set a different error code if we pass a negative key */
                if (state < 0) {
                        ctx->result = ERR_PTR(state);
                        kleave(" = %d [neg]", ctx->skipped_ret);
                        goto skipped;
                }
        }

        /* Found */
        ctx->result = make_key_ref(key, ctx->possessed);
        kleave(" = 1 [found]");
        return 1;

skipped:
        return ctx->skipped_ret;
}

/*
 * Search inside a keyring for a key.  We can search by walking to it
 * directly based on its index-key or we can iterate over the entire
 * tree looking for it, based on the match function.
 */
static int search_keyring(struct key *keyring, struct keyring_search_context *ctx)
{
        if (ctx->match_data.lookup_type == KEYRING_SEARCH_LOOKUP_DIRECT) {
                const void *object;

                object = assoc_array_find(&keyring->keys,
                                          &keyring_assoc_array_ops,
                                          &ctx->index_key);
                return object ? ctx->iterator(object, ctx) : 0;
        }
        return assoc_array_iterate(&keyring->keys, ctx->iterator, ctx);
}

/*
 * Search a tree of keyrings that point to other keyrings up to the maximum
 * depth.
 */
static bool search_nested_keyrings(struct key *keyring,
                                   struct keyring_search_context *ctx)
{
        struct {
                struct key *keyring;
                struct assoc_array_node *node;
                int slot;
        } stack[KEYRING_SEARCH_MAX_DEPTH];

        struct assoc_array_shortcut *shortcut;
        struct assoc_array_node *node;
        struct assoc_array_ptr *ptr;
        struct key *key;
        int sp = 0, slot;

        kenter("{%d},{%s,%s}",
               keyring->serial,
               ctx->index_key.type->name,
               ctx->index_key.description);

#define STATE_CHECKS (KEYRING_SEARCH_NO_STATE_CHECK | KEYRING_SEARCH_DO_STATE_CHECK)
        BUG_ON((ctx->flags & STATE_CHECKS) == 0 ||
               (ctx->flags & STATE_CHECKS) == STATE_CHECKS);

        if (ctx->index_key.description)
                key_set_index_key(&ctx->index_key);

        /* Check to see if this top-level keyring is what we are looking for
         * and whether it is valid or not.
         */
        if (ctx->match_data.lookup_type == KEYRING_SEARCH_LOOKUP_ITERATE ||
            keyring_compare_object(keyring, &ctx->index_key)) {
                ctx->skipped_ret = 2;
                switch (ctx->iterator(keyring_key_to_ptr(keyring), ctx)) {
                case 1:
                        goto found;
                case 2:
                        return false;
                default:
                        break;
                }
        }

        ctx->skipped_ret = 0;

        /* Start processing a new keyring */
descend_to_keyring:
        kdebug("descend to %d", keyring->serial);
        if (keyring->flags & ((1 << KEY_FLAG_INVALIDATED) |
                              (1 << KEY_FLAG_REVOKED)))
                goto not_this_keyring;

        /* Search through the keys in this keyring before its searching its
         * subtrees.
         */
        if (search_keyring(keyring, ctx))
                goto found;

        /* Then manually iterate through the keyrings nested in this one.
         *
         * Start from the root node of the index tree.  Because of the way the
         * hash function has been set up, keyrings cluster on the leftmost
         * branch of the root node (root slot 0) or in the root node itself.
         * Non-keyrings avoid the leftmost branch of the root entirely (root
         * slots 1-15).
         */
        if (!(ctx->flags & KEYRING_SEARCH_RECURSE))
                goto not_this_keyring;

        ptr = READ_ONCE(keyring->keys.root);
        if (!ptr)
                goto not_this_keyring;

        if (assoc_array_ptr_is_shortcut(ptr)) {
                /* If the root is a shortcut, either the keyring only contains
                 * keyring pointers (everything clusters behind root slot 0) or
                 * doesn't contain any keyring pointers.
                 */
                shortcut = assoc_array_ptr_to_shortcut(ptr);
                if ((shortcut->index_key[0] & ASSOC_ARRAY_FAN_MASK) != 0)
                        goto not_this_keyring;

                ptr = READ_ONCE(shortcut->next_node);
                node = assoc_array_ptr_to_node(ptr);
                goto begin_node;
        }

        node = assoc_array_ptr_to_node(ptr);
        ptr = node->slots[0];
        if (!assoc_array_ptr_is_meta(ptr))
                goto begin_node;

descend_to_node:
        /* Descend to a more distal node in this keyring's content tree and go
         * through that.
         */
        kdebug("descend");
        if (assoc_array_ptr_is_shortcut(ptr)) {
                shortcut = assoc_array_ptr_to_shortcut(ptr);
                ptr = READ_ONCE(shortcut->next_node);
                BUG_ON(!assoc_array_ptr_is_node(ptr));
        }
        node = assoc_array_ptr_to_node(ptr);

begin_node:
        kdebug("begin_node");
        slot = 0;
ascend_to_node:
        /* Go through the slots in a node */
        for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
                ptr = READ_ONCE(node->slots[slot]);

                if (assoc_array_ptr_is_meta(ptr)) {
                        if (node->back_pointer ||
                            assoc_array_ptr_is_shortcut(ptr))
                                goto descend_to_node;
                }

                if (!keyring_ptr_is_keyring(ptr))
                        continue;

                key = keyring_ptr_to_key(ptr);

                if (sp >= KEYRING_SEARCH_MAX_DEPTH) {
                        if (ctx->flags & KEYRING_SEARCH_DETECT_TOO_DEEP) {
                                ctx->result = ERR_PTR(-ELOOP);
                                return false;
                        }
                        goto not_this_keyring;
                }

                /* Search a nested keyring */
                if (!(ctx->flags & KEYRING_SEARCH_NO_CHECK_PERM) &&
                    key_task_permission(make_key_ref(key, ctx->possessed),
                                        ctx->cred, KEY_NEED_SEARCH) < 0)
                        continue;

                /* stack the current position */
                stack[sp].keyring = keyring;
                stack[sp].node = node;
                stack[sp].slot = slot;
                sp++;

                /* begin again with the new keyring */
                keyring = key;
                goto descend_to_keyring;
        }

        /* We've dealt with all the slots in the current node, so now we need
         * to ascend to the parent and continue processing there.
         */
        ptr = READ_ONCE(node->back_pointer);
        slot = node->parent_slot;

        if (ptr && assoc_array_ptr_is_shortcut(ptr)) {
                shortcut = assoc_array_ptr_to_shortcut(ptr);
                ptr = READ_ONCE(shortcut->back_pointer);
                slot = shortcut->parent_slot;
        }
        if (!ptr)
                goto not_this_keyring;
        node = assoc_array_ptr_to_node(ptr);
        slot++;

        /* If we've ascended to the root (zero backpointer), we must have just
         * finished processing the leftmost branch rather than the root slots -
         * so there can't be any more keyrings for us to find.
         */
        if (node->back_pointer) {
                kdebug("ascend %d", slot);
                goto ascend_to_node;
        }

        /* The keyring we're looking at was disqualified or didn't contain a
         * matching key.
         */
not_this_keyring:
        kdebug("not_this_keyring %d", sp);
        if (sp <= 0) {
                kleave(" = false");
                return false;
        }

        /* Resume the processing of a keyring higher up in the tree */
        sp--;
        keyring = stack[sp].keyring;
        node = stack[sp].node;
        slot = stack[sp].slot + 1;
        kdebug("ascend to %d [%d]", keyring->serial, slot);
        goto ascend_to_node;

        /* We found a viable match */
found:
        key = key_ref_to_ptr(ctx->result);
        key_check(key);
        if (!(ctx->flags & KEYRING_SEARCH_NO_UPDATE_TIME)) {
                key->last_used_at = ctx->now;
                keyring->last_used_at = ctx->now;
                while (sp > 0)
                        stack[--sp].keyring->last_used_at = ctx->now;
        }
        kleave(" = true");
        return true;
}

/**
 * keyring_search_rcu - Search a keyring tree for a matching key under RCU
 * @keyring_ref: A pointer to the keyring with possession indicator.
 * @ctx: The keyring search context.
 *
 * Search the supplied keyring tree for a key that matches the criteria given.
 * The root keyring and any linked keyrings must grant Search permission to the
 * caller to be searchable and keys can only be found if they too grant Search
 * to the caller. The possession flag on the root keyring pointer controls use
 * of the possessor bits in permissions checking of the entire tree.  In
 * addition, the LSM gets to forbid keyring searches and key matches.
 *
 * The search is performed as a breadth-then-depth search up to the prescribed
 * limit (KEYRING_SEARCH_MAX_DEPTH).  The caller must hold the RCU read lock to
 * prevent keyrings from being destroyed or rearranged whilst they are being
 * searched.
 *
 * Keys are matched to the type provided and are then filtered by the match
 * function, which is given the description to use in any way it sees fit.  The
 * match function may use any attributes of a key that it wishes to
 * determine the match.  Normally the match function from the key type would be
 * used.
 *
 * RCU can be used to prevent the keyring key lists from disappearing without
 * the need to take lots of locks.
 *
 * Returns a pointer to the found key and increments the key usage count if
 * successful; -EAGAIN if no matching keys were found, or if expired or revoked
 * keys were found; -ENOKEY if only negative keys were found; -ENOTDIR if the
 * specified keyring wasn't a keyring.
 *
 * In the case of a successful return, the possession attribute from
 * @keyring_ref is propagated to the returned key reference.
 */
key_ref_t keyring_search_rcu(key_ref_t keyring_ref,
                             struct keyring_search_context *ctx)
{
        struct key *keyring;
        long err;

        ctx->iterator = keyring_search_iterator;
        ctx->possessed = is_key_possessed(keyring_ref);
        ctx->result = ERR_PTR(-EAGAIN);

        keyring = key_ref_to_ptr(keyring_ref);
        key_check(keyring);

        if (keyring->type != &key_type_keyring)
                return ERR_PTR(-ENOTDIR);

        if (!(ctx->flags & KEYRING_SEARCH_NO_CHECK_PERM)) {
                err = key_task_permission(keyring_ref, ctx->cred, KEY_NEED_SEARCH);
                if (err < 0)
                        return ERR_PTR(err);
        }

        ctx->now = ktime_get_real_seconds();
        if (search_nested_keyrings(keyring, ctx))
                __key_get(key_ref_to_ptr(ctx->result));
        return ctx->result;
}

/**
 * keyring_search - Search the supplied keyring tree for a matching key
 * @keyring: The root of the keyring tree to be searched.
 * @type: The type of keyring we want to find.
 * @description: The name of the keyring we want to find.
 * @recurse: True to search the children of @keyring also
 *
 * As keyring_search_rcu() above, but using the current task's credentials and
 * type's default matching function and preferred search method.
 */
key_ref_t keyring_search(key_ref_t keyring,
                         struct key_type *type,
                         const char *description,
                         bool recurse)
{
        struct keyring_search_context ctx = {
                .index_key.type                = type,
                .index_key.description        = description,
                .index_key.desc_len        = strlen(description),
                .cred                        = current_cred(),
                .match_data.cmp                = key_default_cmp,
                .match_data.raw_data        = description,
                .match_data.lookup_type        = KEYRING_SEARCH_LOOKUP_DIRECT,
                .flags                        = KEYRING_SEARCH_DO_STATE_CHECK,
        };
        key_ref_t key;
        int ret;

        if (recurse)
                ctx.flags |= KEYRING_SEARCH_RECURSE;
        if (type->match_preparse) {
                ret = type->match_preparse(&ctx.match_data);
                if (ret < 0)
                        return ERR_PTR(ret);
        }

        rcu_read_lock();
        key = keyring_search_rcu(keyring, &ctx);
        rcu_read_unlock();

        if (type->match_free)
                type->match_free(&ctx.match_data);
        return key;
}
EXPORT_SYMBOL(keyring_search);

static struct key_restriction *keyring_restriction_alloc(
        key_restrict_link_func_t check)
{
        struct key_restriction *keyres =
                kzalloc(sizeof(struct key_restriction), GFP_KERNEL);

        if (!keyres)
                return ERR_PTR(-ENOMEM);

        keyres->check = check;

        return keyres;
}

/*
 * Semaphore to serialise restriction setup to prevent reference count
 * cycles through restriction key pointers.
 */
static DECLARE_RWSEM(keyring_serialise_restrict_sem);

/*
 * Check for restriction cycles that would prevent keyring garbage collection.
 * keyring_serialise_restrict_sem must be held.
 */
static bool keyring_detect_restriction_cycle(const struct key *dest_keyring,
                                             struct key_restriction *keyres)
{
        while (keyres && keyres->key &&
               keyres->key->type == &key_type_keyring) {
                if (keyres->key == dest_keyring)
                        return true;

                keyres = keyres->key->restrict_link;
        }

        return false;
}

/**
 * keyring_restrict - Look up and apply a restriction to a keyring
 * @keyring_ref: The keyring to be restricted
 * @type: The key type that will provide the restriction checker.
 * @restriction: The restriction options to apply to the keyring
 *
 * Look up a keyring and apply a restriction to it.  The restriction is managed
 * by the specific key type, but can be configured by the options specified in
 * the restriction string.
 */
int keyring_restrict(key_ref_t keyring_ref, const char *type,
                     const char *restriction)
{
        struct key *keyring;
        struct key_type *restrict_type = NULL;
        struct key_restriction *restrict_link;
        int ret = 0;

        keyring = key_ref_to_ptr(keyring_ref);
        key_check(keyring);

        if (keyring->type != &key_type_keyring)
                return -ENOTDIR;

        if (!type) {
                restrict_link = keyring_restriction_alloc(restrict_link_reject);
        } else {
                restrict_type = key_type_lookup(type);

                if (IS_ERR(restrict_type))
                        return PTR_ERR(restrict_type);

                if (!restrict_type->lookup_restriction) {
                        ret = -ENOENT;
                        goto error;
                }

                restrict_link = restrict_type->lookup_restriction(restriction);
        }

        if (IS_ERR(restrict_link)) {
                ret = PTR_ERR(restrict_link);
                goto error;
        }

        down_write(&keyring->sem);
        down_write(&keyring_serialise_restrict_sem);

        if (keyring->restrict_link) {
                ret = -EEXIST;
        } else if (keyring_detect_restriction_cycle(keyring, restrict_link)) {
                ret = -EDEADLK;
        } else {
                keyring->restrict_link = restrict_link;
                notify_key(keyring, NOTIFY_KEY_SETATTR, 0);
        }

        up_write(&keyring_serialise_restrict_sem);
        up_write(&keyring->sem);

        if (ret < 0) {
                key_put(restrict_link->key);
                kfree(restrict_link);
        }

error:
        if (restrict_type)
                key_type_put(restrict_type);

        return ret;
}
EXPORT_SYMBOL(keyring_restrict);

/*
 * Search the given keyring for a key that might be updated.
 *
 * The caller must guarantee that the keyring is a keyring and that the
 * permission is granted to modify the keyring as no check is made here.  The
 * caller must also hold a lock on the keyring semaphore.
 *
 * Returns a pointer to the found key with usage count incremented if
 * successful and returns NULL if not found.  Revoked and invalidated keys are
 * skipped over.
 *
 * If successful, the possession indicator is propagated from the keyring ref
 * to the returned key reference.
 */
key_ref_t find_key_to_update(key_ref_t keyring_ref,
                             const struct keyring_index_key *index_key)
{
        struct key *keyring, *key;
        const void *object;

        keyring = key_ref_to_ptr(keyring_ref);

        kenter("{%d},{%s,%s}",
               keyring->serial, index_key->type->name, index_key->description);

        object = assoc_array_find(&keyring->keys, &keyring_assoc_array_ops,
                                  index_key);

        if (object)
                goto found;

        kleave(" = NULL");
        return NULL;

found:
        key = keyring_ptr_to_key(object);
        if (key->flags & ((1 << KEY_FLAG_INVALIDATED) |
                          (1 << KEY_FLAG_REVOKED))) {
                kleave(" = NULL [x]");
                return NULL;
        }
        __key_get(key);
        kleave(" = {%d}", key->serial);
        return make_key_ref(key, is_key_possessed(keyring_ref));
}

/*
 * Find a keyring with the specified name.
 *
 * Only keyrings that have nonzero refcount, are not revoked, and are owned by a
 * user in the current user namespace are considered.  If @uid_keyring is %true,
 * the keyring additionally must have been allocated as a user or user session
 * keyring; otherwise, it must grant Search permission directly to the caller.
 *
 * Returns a pointer to the keyring with the keyring's refcount having being
 * incremented on success.  -ENOKEY is returned if a key could not be found.
 */
struct key *find_keyring_by_name(const char *name, bool uid_keyring)
{
        struct user_namespace *ns = current_user_ns();
        struct key *keyring;

        if (!name)
                return ERR_PTR(-EINVAL);

        read_lock(&keyring_name_lock);

        /* Search this hash bucket for a keyring with a matching name that
         * grants Search permission and that hasn't been revoked
         */
        list_for_each_entry(keyring, &ns->keyring_name_list, name_link) {
                if (!kuid_has_mapping(ns, keyring->user->uid))
                        continue;

                if (test_bit(KEY_FLAG_REVOKED, &keyring->flags))
                        continue;

                if (strcmp(keyring->description, name) != 0)
                        continue;

                if (uid_keyring) {
                        if (!test_bit(KEY_FLAG_UID_KEYRING,
                                      &keyring->flags))
                                continue;
                } else {
                        if (key_permission(make_key_ref(keyring, 0),
                                           KEY_NEED_SEARCH) < 0)
                                continue;
                }

                /* we've got a match but we might end up racing with
                 * key_cleanup() if the keyring is currently 'dead'
                 * (ie. it has a zero usage count) */
                if (!refcount_inc_not_zero(&keyring->usage))
                        continue;
                keyring->last_used_at = ktime_get_real_seconds();
                goto out;
        }

        keyring = ERR_PTR(-ENOKEY);
out:
        read_unlock(&keyring_name_lock);
        return keyring;
}

static int keyring_detect_cycle_iterator(const void *object,
                                         void *iterator_data)
{
        struct keyring_search_context *ctx = iterator_data;
        const struct key *key = keyring_ptr_to_key(object);

        kenter("{%d}", key->serial);

        /* We might get a keyring with matching index-key that is nonetheless a
         * different keyring. */
        if (key != ctx->match_data.raw_data)
                return 0;

        ctx->result = ERR_PTR(-EDEADLK);
        return 1;
}

/*
 * See if a cycle will be created by inserting acyclic tree B in acyclic
 * tree A at the topmost level (ie: as a direct child of A).
 *
 * Since we are adding B to A at the top level, checking for cycles should just
 * be a matter of seeing if node A is somewhere in tree B.
 */
static int keyring_detect_cycle(struct key *A, struct key *B)
{
        struct keyring_search_context ctx = {
                .index_key                = A->index_key,
                .match_data.raw_data        = A,
                .match_data.lookup_type = KEYRING_SEARCH_LOOKUP_DIRECT,
                .iterator                = keyring_detect_cycle_iterator,
                .flags                        = (KEYRING_SEARCH_NO_STATE_CHECK |
                                           KEYRING_SEARCH_NO_UPDATE_TIME |
                                           KEYRING_SEARCH_NO_CHECK_PERM |
                                           KEYRING_SEARCH_DETECT_TOO_DEEP |
                                           KEYRING_SEARCH_RECURSE),
        };

        rcu_read_lock();
        search_nested_keyrings(B, &ctx);
        rcu_read_unlock();
        return PTR_ERR(ctx.result) == -EAGAIN ? 0 : PTR_ERR(ctx.result);
}

/*
 * Lock keyring for link.
 */
int __key_link_lock(struct key *keyring,
                    const struct keyring_index_key *index_key)
        __acquires(&keyring->sem)
        __acquires(&keyring_serialise_link_lock)
{
        if (keyring->type != &key_type_keyring)
                return -ENOTDIR;

        down_write(&keyring->sem);

        /* Serialise link/link calls to prevent parallel calls causing a cycle
         * when linking two keyring in opposite orders.
         */
        if (index_key->type == &key_type_keyring)
                mutex_lock(&keyring_serialise_link_lock);

        return 0;
}

/*
 * Lock keyrings for move (link/unlink combination).
 */
int __key_move_lock(struct key *l_keyring, struct key *u_keyring,
                    const struct keyring_index_key *index_key)
        __acquires(&l_keyring->sem)
        __acquires(&u_keyring->sem)
        __acquires(&keyring_serialise_link_lock)
{
        if (l_keyring->type != &key_type_keyring ||
            u_keyring->type != &key_type_keyring)
                return -ENOTDIR;

        /* We have to be very careful here to take the keyring locks in the
         * right order, lest we open ourselves to deadlocking against another
         * move operation.
         */
        if (l_keyring < u_keyring) {
                down_write(&l_keyring->sem);
                down_write_nested(&u_keyring->sem, 1);
        } else {
                down_write(&u_keyring->sem);
                down_write_nested(&l_keyring->sem, 1);
        }

        /* Serialise link/link calls to prevent parallel calls causing a cycle
         * when linking two keyring in opposite orders.
         */
        if (index_key->type == &key_type_keyring)
                mutex_lock(&keyring_serialise_link_lock);

        return 0;
}

/*
 * Preallocate memory so that a key can be linked into to a keyring.
 */
int __key_link_begin(struct key *keyring,
                     const struct keyring_index_key *index_key,
                     struct assoc_array_edit **_edit)
{
        struct assoc_array_edit *edit;
        int ret;

        kenter("%d,%s,%s,",
               keyring->serial, index_key->type->name, index_key->description);

        BUG_ON(index_key->desc_len == 0);
        BUG_ON(*_edit != NULL);

        *_edit = NULL;

        ret = -EKEYREVOKED;
        if (test_bit(KEY_FLAG_REVOKED, &keyring->flags))
                goto error;

        /* Create an edit script that will insert/replace the key in the
         * keyring tree.
         */
        edit = assoc_array_insert(&keyring->keys,
                                  &keyring_assoc_array_ops,
                                  index_key,
                                  NULL);
        if (IS_ERR(edit)) {
                ret = PTR_ERR(edit);
                goto error;
        }

        /* If we're not replacing a link in-place then we're going to need some
         * extra quota.
         */
        if (!edit->dead_leaf) {
                ret = key_payload_reserve(keyring,
                                          keyring->datalen + KEYQUOTA_LINK_BYTES);
                if (ret < 0)
                        goto error_cancel;
        }

        *_edit = edit;
        kleave(" = 0");
        return 0;

error_cancel:
        assoc_array_cancel_edit(edit);
error:
        kleave(" = %d", ret);
        return ret;
}

/*
 * Check already instantiated keys aren't going to be a problem.
 *
 * The caller must have called __key_link_begin(). Don't need to call this for
 * keys that were created since __key_link_begin() was called.
 */
int __key_link_check_live_key(struct key *keyring, struct key *key)
{
        if (key->type == &key_type_keyring)
                /* check that we aren't going to create a cycle by linking one
                 * keyring to another */
                return keyring_detect_cycle(keyring, key);
        return 0;
}

/*
 * Link a key into to a keyring.
 *
 * Must be called with __key_link_begin() having being called.  Discards any
 * already extant link to matching key if there is one, so that each keyring
 * holds at most one link to any given key of a particular type+description
 * combination.
 */
void __key_link(struct key *keyring, struct key *key,
                struct assoc_array_edit **_edit)
{
        __key_get(key);
        assoc_array_insert_set_object(*_edit, keyring_key_to_ptr(key));
        assoc_array_apply_edit(*_edit);
        *_edit = NULL;
        notify_key(keyring, NOTIFY_KEY_LINKED, key_serial(key));
}

/*
 * Finish linking a key into to a keyring.
 *
 * Must be called with __key_link_begin() having being called.
 */
void __key_link_end(struct key *keyring,
                    const struct keyring_index_key *index_key,
                    struct assoc_array_edit *edit)
        __releases(&keyring->sem)
        __releases(&keyring_serialise_link_lock)
{
        BUG_ON(index_key->type == NULL);
        kenter("%d,%s,", keyring->serial, index_key->type->name);

        if (edit) {
                if (!edit->dead_leaf) {
                        key_payload_reserve(keyring,
                                keyring->datalen - KEYQUOTA_LINK_BYTES);
                }
                assoc_array_cancel_edit(edit);
        }
        up_write(&keyring->sem);

        if (index_key->type == &key_type_keyring)
                mutex_unlock(&keyring_serialise_link_lock);
}

/*
 * Check addition of keys to restricted keyrings.
 */
static int __key_link_check_restriction(struct key *keyring, struct key *key)
{
        if (!keyring->restrict_link || !keyring->restrict_link->check)
                return 0;
        return keyring->restrict_link->check(keyring, key->type, &key->payload,
                                             keyring->restrict_link->key);
}

/**
 * key_link - Link a key to a keyring
 * @keyring: The keyring to make the link in.
 * @key: The key to link to.
 *
 * Make a link in a keyring to a key, such that the keyring holds a reference
 * on that key and the key can potentially be found by searching that keyring.
 *
 * This function will write-lock the keyring's semaphore and will consume some
 * of the user's key data quota to hold the link.
 *
 * Returns 0 if successful, -ENOTDIR if the keyring isn't a keyring,
 * -EKEYREVOKED if the keyring has been revoked, -ENFILE if the keyring is
 * full, -EDQUOT if there is insufficient key data quota remaining to add
 * another link or -ENOMEM if there's insufficient memory.
 *
 * It is assumed that the caller has checked that it is permitted for a link to
 * be made (the keyring should have Write permission and the key Link
 * permission).
 */
int key_link(struct key *keyring, struct key *key)
{
        struct assoc_array_edit *edit = NULL;
        int ret;

        kenter("{%d,%d}", keyring->serial, refcount_read(&keyring->usage));

        key_check(keyring);
        key_check(key);

        ret = __key_link_lock(keyring, &key->index_key);
        if (ret < 0)
                goto error;

        ret = __key_link_begin(keyring, &key->index_key, &edit);
        if (ret < 0)
                goto error_end;

        kdebug("begun {%d,%d}", keyring->serial, refcount_read(&keyring->usage));
        ret = __key_link_check_restriction(keyring, key);
        if (ret == 0)
                ret = __key_link_check_live_key(keyring, key);
        if (ret == 0)
                __key_link(keyring, key, &edit);

error_end:
        __key_link_end(keyring, &key->index_key, edit);
error:
        kleave(" = %d {%d,%d}", ret, keyring->serial, refcount_read(&keyring->usage));
        return ret;
}
EXPORT_SYMBOL(key_link);

/*
 * Lock a keyring for unlink.
 */
static int __key_unlink_lock(struct key *keyring)
        __acquires(&keyring->sem)
{
        if (keyring->type != &key_type_keyring)
                return -ENOTDIR;

        down_write(&keyring->sem);
        return 0;
}

/*
 * Begin the process of unlinking a key from a keyring.
 */
static int __key_unlink_begin(struct key *keyring, struct key *key,
                              struct assoc_array_edit **_edit)
{
        struct assoc_array_edit *edit;

        BUG_ON(*_edit != NULL);

        edit = assoc_array_delete(&keyring->keys, &keyring_assoc_array_ops,
                                  &key->index_key);
        if (IS_ERR(edit))
                return PTR_ERR(edit);

        if (!edit)
                return -ENOENT;

        *_edit = edit;
        return 0;
}

/*
 * Apply an unlink change.
 */
static void __key_unlink(struct key *keyring, struct key *key,
                         struct assoc_array_edit **_edit)
{
        assoc_array_apply_edit(*_edit);
        notify_key(keyring, NOTIFY_KEY_UNLINKED, key_serial(key));
        *_edit = NULL;
        key_payload_reserve(keyring, keyring->datalen - KEYQUOTA_LINK_BYTES);
}

/*
 * Finish unlinking a key from to a keyring.
 */
static void __key_unlink_end(struct key *keyring,
                             struct key *key,
                             struct assoc_array_edit *edit)
        __releases(&keyring->sem)
{
        if (edit)
                assoc_array_cancel_edit(edit);
        up_write(&keyring->sem);
}

/**
 * key_unlink - Unlink the first link to a key from a keyring.
 * @keyring: The keyring to remove the link from.
 * @key: The key the link is to.
 *
 * Remove a link from a keyring to a key.
 *
 * This function will write-lock the keyring's semaphore.
 *
 * Returns 0 if successful, -ENOTDIR if the keyring isn't a keyring, -ENOENT if
 * the key isn't linked to by the keyring or -ENOMEM if there's insufficient
 * memory.
 *
 * It is assumed that the caller has checked that it is permitted for a link to
 * be removed (the keyring should have Write permission; no permissions are
 * required on the key).
 */
int key_unlink(struct key *keyring, struct key *key)
{
        struct assoc_array_edit *edit = NULL;
        int ret;

        key_check(keyring);
        key_check(key);

        ret = __key_unlink_lock(keyring);
        if (ret < 0)
                return ret;

        ret = __key_unlink_begin(keyring, key, &edit);
        if (ret == 0)
                __key_unlink(keyring, key, &edit);
        __key_unlink_end(keyring, key, edit);
        return ret;
}
EXPORT_SYMBOL(key_unlink);

/**
 * key_move - Move a key from one keyring to another
 * @key: The key to move
 * @from_keyring: The keyring to remove the link from.
 * @to_keyring: The keyring to make the link in.
 * @flags: Qualifying flags, such as KEYCTL_MOVE_EXCL.
 *
 * Make a link in @to_keyring to a key, such that the keyring holds a reference
 * on that key and the key can potentially be found by searching that keyring
 * whilst simultaneously removing a link to the key from @from_keyring.
 *
 * This function will write-lock both keyring's semaphores and will consume
 * some of the user's key data quota to hold the link on @to_keyring.
 *
 * Returns 0 if successful, -ENOTDIR if either keyring isn't a keyring,
 * -EKEYREVOKED if either keyring has been revoked, -ENFILE if the second
 * keyring is full, -EDQUOT if there is insufficient key data quota remaining
 * to add another link or -ENOMEM if there's insufficient memory.  If
 * KEYCTL_MOVE_EXCL is set, then -EEXIST will be returned if there's already a
 * matching key in @to_keyring.
 *
 * It is assumed that the caller has checked that it is permitted for a link to
 * be made (the keyring should have Write permission and the key Link
 * permission).
 */
int key_move(struct key *key,
             struct key *from_keyring,
             struct key *to_keyring,
             unsigned int flags)
{
        struct assoc_array_edit *from_edit = NULL, *to_edit = NULL;
        int ret;

        kenter("%d,%d,%d", key->serial, from_keyring->serial, to_keyring->serial);

        if (from_keyring == to_keyring)
                return 0;

        key_check(key);
        key_check(from_keyring);
        key_check(to_keyring);

        ret = __key_move_lock(from_keyring, to_keyring, &key->index_key);
        if (ret < 0)
                goto out;
        ret = __key_unlink_begin(from_keyring, key, &from_edit);
        if (ret < 0)
                goto error;
        ret = __key_link_begin(to_keyring, &key->index_key, &to_edit);
        if (ret < 0)
                goto error;

        ret = -EEXIST;
        if (to_edit->dead_leaf && (flags & KEYCTL_MOVE_EXCL))
                goto error;

        ret = __key_link_check_restriction(to_keyring, key);
        if (ret < 0)
                goto error;
        ret = __key_link_check_live_key(to_keyring, key);
        if (ret < 0)
                goto error;

        __key_unlink(from_keyring, key, &from_edit);
        __key_link(to_keyring, key, &to_edit);
error:
        __key_link_end(to_keyring, &key->index_key, to_edit);
        __key_unlink_end(from_keyring, key, from_edit);
out:
        kleave(" = %d", ret);
        return ret;
}
EXPORT_SYMBOL(key_move);

/**
 * keyring_clear - Clear a keyring
 * @keyring: The keyring to clear.
 *
 * Clear the contents of the specified keyring.
 *
 * Returns 0 if successful or -ENOTDIR if the keyring isn't a keyring.
 */
int keyring_clear(struct key *keyring)
{
        struct assoc_array_edit *edit;
        int ret;

        if (keyring->type != &key_type_keyring)
                return -ENOTDIR;

        down_write(&keyring->sem);

        edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops);
        if (IS_ERR(edit)) {
                ret = PTR_ERR(edit);
        } else {
                if (edit)
                        assoc_array_apply_edit(edit);
                notify_key(keyring, NOTIFY_KEY_CLEARED, 0);
                key_payload_reserve(keyring, 0);
                ret = 0;
        }

        up_write(&keyring->sem);
        return ret;
}
EXPORT_SYMBOL(keyring_clear);

/*
 * Dispose of the links from a revoked keyring.
 *
 * This is called with the key sem write-locked.
 */
static void keyring_revoke(struct key *keyring)
{
        struct assoc_array_edit *edit;

        edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops);
        if (!IS_ERR(edit)) {
                if (edit)
                        assoc_array_apply_edit(edit);
                key_payload_reserve(keyring, 0);
        }
}

static bool keyring_gc_select_iterator(void *object, void *iterator_data)
{
        struct key *key = keyring_ptr_to_key(object);
        time64_t *limit = iterator_data;

        if (key_is_dead(key, *limit))
                return false;
        key_get(key);
        return true;
}

static int keyring_gc_check_iterator(const void *object, void *iterator_data)
{
        const struct key *key = keyring_ptr_to_key(object);
        time64_t *limit = iterator_data;

        key_check(key);
        return key_is_dead(key, *limit);
}

/*
 * Garbage collect pointers from a keyring.
 *
 * Not called with any locks held.  The keyring's key struct will not be
 * deallocated under us as only our caller may deallocate it.
 */
void keyring_gc(struct key *keyring, time64_t limit)
{
        int result;

        kenter("%x{%s}", keyring->serial, keyring->description ?: "");

        if (keyring->flags & ((1 << KEY_FLAG_INVALIDATED) |
                              (1 << KEY_FLAG_REVOKED)))
                goto dont_gc;

        /* scan the keyring looking for dead keys */
        rcu_read_lock();
        result = assoc_array_iterate(&keyring->keys,
                                     keyring_gc_check_iterator, &limit);
        rcu_read_unlock();
        if (result == true)
                goto do_gc;

dont_gc:
        kleave(" [no gc]");
        return;

do_gc:
        down_write(&keyring->sem);
        assoc_array_gc(&keyring->keys, &keyring_assoc_array_ops,
                       keyring_gc_select_iterator, &limit);
        up_write(&keyring->sem);
        kleave(" [gc]");
}

/*
 * Garbage collect restriction pointers from a keyring.
 *
 * Keyring restrictions are associated with a key type, and must be cleaned
 * up if the key type is unregistered. The restriction is altered to always
 * reject additional keys so a keyring cannot be opened up by unregistering
 * a key type.
 *
 * Not called with any keyring locks held. The keyring's key struct will not
 * be deallocated under us as only our caller may deallocate it.
 *
 * The caller is required to hold key_types_sem and dead_type->sem. This is
 * fulfilled by key_gc_keytype() holding the locks on behalf of
 * key_garbage_collector(), which it invokes on a workqueue.
 */
void keyring_restriction_gc(struct key *keyring, struct key_type *dead_type)
{
        struct key_restriction *keyres;

        kenter("%x{%s}", keyring->serial, keyring->description ?: "");

        /*
         * keyring->restrict_link is only assigned at key allocation time
         * or with the key type locked, so the only values that could be
         * concurrently assigned to keyring->restrict_link are for key
         * types other than dead_type. Given this, it's ok to check
         * the key type before acquiring keyring->sem.
         */
        if (!dead_type || !keyring->restrict_link ||
            keyring->restrict_link->keytype != dead_type) {
                kleave(" [no restriction gc]");
                return;
        }

        /* Lock the keyring to ensure that a link is not in progress */
        down_write(&keyring->sem);

        keyres = keyring->restrict_link;

        keyres->check = restrict_link_reject;

        key_put(keyres->key);
        keyres->key = NULL;
        keyres->keytype = NULL;

        up_write(&keyring->sem);

        kleave(" [restriction gc]");
}





































































































































































   27 














   13 

















































 1255 































































































































































































   12 













   11 


   25 
    2 
   23 













  213 


   31 
   20 
   11 







  213 

























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_FILELOCK_H
#define _LINUX_FILELOCK_H

#include <linux/fs.h>

#define FL_POSIX        1
#define FL_FLOCK        2
#define FL_DELEG        4        /* NFSv4 delegation */
#define FL_ACCESS        8        /* not trying to lock, just looking */
#define FL_EXISTS        16        /* when unlocking, test for existence */
#define FL_LEASE        32        /* lease held on this file */
#define FL_CLOSE        64        /* unlock on close */
#define FL_SLEEP        128        /* A blocking lock */
#define FL_DOWNGRADE_PENDING        256 /* Lease is being downgraded */
#define FL_UNLOCK_PENDING        512 /* Lease is being broken */
#define FL_OFDLCK        1024        /* lock is "owned" by struct file */
#define FL_LAYOUT        2048        /* outstanding pNFS layout */
#define FL_RECLAIM        4096        /* reclaiming from a reboot server */

#define FL_CLOSE_POSIX (FL_POSIX | FL_CLOSE)

/*
 * Special return value from posix_lock_file() and vfs_lock_file() for
 * asynchronous locking.
 */
#define FILE_LOCK_DEFERRED 1

struct file_lock;
struct file_lease;

struct file_lock_operations {
        void (*fl_copy_lock)(struct file_lock *, struct file_lock *);
        void (*fl_release_private)(struct file_lock *);
};

struct lock_manager_operations {
        void *lm_mod_owner;
        fl_owner_t (*lm_get_owner)(fl_owner_t);
        void (*lm_put_owner)(fl_owner_t);
        void (*lm_notify)(struct file_lock *);        /* unblock callback */
        int (*lm_grant)(struct file_lock *, int);
        bool (*lm_lock_expirable)(struct file_lock *cfl);
        void (*lm_expire_lock)(void);
};

struct lease_manager_operations {
        bool (*lm_break)(struct file_lease *);
        int (*lm_change)(struct file_lease *, int, struct list_head *);
        void (*lm_setup)(struct file_lease *, void **);
        bool (*lm_breaker_owns_lease)(struct file_lease *);
};

struct lock_manager {
        struct list_head list;
        /*
         * NFSv4 and up also want opens blocked during the grace period;
         * NLM doesn't care:
         */
        bool block_opens;
};

struct net;
void locks_start_grace(struct net *, struct lock_manager *);
void locks_end_grace(struct lock_manager *);
bool locks_in_grace(struct net *);
bool opens_in_grace(struct net *);

/*
 * struct file_lock has a union that some filesystems use to track
 * their own private info. The NFS side of things is defined here:
 */
#include <linux/nfs_fs_i.h>

/*
 * struct file_lock represents a generic "file lock". It's used to represent
 * POSIX byte range locks, BSD (flock) locks, and leases. It's important to
 * note that the same struct is used to represent both a request for a lock and
 * the lock itself, but the same object is never used for both.
 *
 * FIXME: should we create a separate "struct lock_request" to help distinguish
 * these two uses?
 *
 * The varous i_flctx lists are ordered by:
 *
 * 1) lock owner
 * 2) lock range start
 * 3) lock range end
 *
 * Obviously, the last two criteria only matter for POSIX locks.
 */

struct file_lock_core {
        struct file_lock_core *flc_blocker;        /* The lock that is blocking us */
        struct list_head flc_list;        /* link into file_lock_context */
        struct hlist_node flc_link;        /* node in global lists */
        struct list_head flc_blocked_requests;        /* list of requests with
                                                 * ->fl_blocker pointing here
                                                 */
        struct list_head flc_blocked_member;        /* node in
                                                 * ->fl_blocker->fl_blocked_requests
                                                 */
        fl_owner_t flc_owner;
        unsigned int flc_flags;
        unsigned char flc_type;
        pid_t flc_pid;
        int flc_link_cpu;                /* what cpu's list is this on? */
        wait_queue_head_t flc_wait;
        struct file *flc_file;
};

struct file_lock {
        struct file_lock_core c;
        loff_t fl_start;
        loff_t fl_end;

        const struct file_lock_operations *fl_ops;        /* Callbacks for filesystems */
        const struct lock_manager_operations *fl_lmops;        /* Callbacks for lockmanagers */
        union {
                struct nfs_lock_info        nfs_fl;
                struct nfs4_lock_info        nfs4_fl;
                struct {
                        struct list_head link;        /* link in AFS vnode's pending_locks list */
                        int state;                /* state of grant or error if -ve */
                        unsigned int        debug_id;
                } afs;
                struct {
                        struct inode *inode;
                } ceph;
        } fl_u;
} __randomize_layout;

struct file_lease {
        struct file_lock_core c;
        struct fasync_struct *        fl_fasync; /* for lease break notifications */
        /* for lease breaks: */
        unsigned long fl_break_time;
        unsigned long fl_downgrade_time;
        const struct lease_manager_operations *fl_lmops; /* Callbacks for lease managers */
} __randomize_layout;

struct file_lock_context {
        spinlock_t                flc_lock;
        struct list_head        flc_flock;
        struct list_head        flc_posix;
        struct list_head        flc_lease;
};

#ifdef CONFIG_FILE_LOCKING
int fcntl_getlk(struct file *, unsigned int, struct flock *);
int fcntl_setlk(unsigned int, struct file *, unsigned int,
                        struct flock *);

#if BITS_PER_LONG == 32
int fcntl_getlk64(struct file *, unsigned int, struct flock64 *);
int fcntl_setlk64(unsigned int, struct file *, unsigned int,
                        struct flock64 *);
#endif

int fcntl_setlease(unsigned int fd, struct file *filp, int arg);
int fcntl_getlease(struct file *filp);

static inline bool lock_is_unlock(struct file_lock *fl)
{
        return fl->c.flc_type == F_UNLCK;
}

static inline bool lock_is_read(struct file_lock *fl)
{
        return fl->c.flc_type == F_RDLCK;
}

static inline bool lock_is_write(struct file_lock *fl)
{
        return fl->c.flc_type == F_WRLCK;
}

static inline void locks_wake_up(struct file_lock *fl)
{
        wake_up(&fl->c.flc_wait);
}

static inline bool locks_can_async_lock(const struct file_operations *fops)
{
        return !fops->lock || fops->fop_flags & FOP_ASYNC_LOCK;
}

/* fs/locks.c */
void locks_free_lock_context(struct inode *inode);
void locks_free_lock(struct file_lock *fl);
void locks_init_lock(struct file_lock *);
struct file_lock *locks_alloc_lock(void);
void locks_copy_lock(struct file_lock *, struct file_lock *);
void locks_copy_conflock(struct file_lock *, struct file_lock *);
void locks_remove_posix(struct file *, fl_owner_t);
void locks_remove_file(struct file *);
void locks_release_private(struct file_lock *);
void posix_test_lock(struct file *, struct file_lock *);
int posix_lock_file(struct file *, struct file_lock *, struct file_lock *);
int locks_delete_block(struct file_lock *);
int vfs_test_lock(struct file *, struct file_lock *);
int vfs_lock_file(struct file *, unsigned int, struct file_lock *, struct file_lock *);
int vfs_cancel_lock(struct file *filp, struct file_lock *fl);
bool vfs_inode_has_locks(struct inode *inode);
int locks_lock_inode_wait(struct inode *inode, struct file_lock *fl);

void locks_init_lease(struct file_lease *);
void locks_free_lease(struct file_lease *fl);
struct file_lease *locks_alloc_lease(void);
int __break_lease(struct inode *inode, unsigned int flags, unsigned int type);
void lease_get_mtime(struct inode *, struct timespec64 *time);
int generic_setlease(struct file *, int, struct file_lease **, void **priv);
int kernel_setlease(struct file *, int, struct file_lease **, void **);
int vfs_setlease(struct file *, int, struct file_lease **, void **);
int lease_modify(struct file_lease *, int, struct list_head *);

struct notifier_block;
int lease_register_notifier(struct notifier_block *);
void lease_unregister_notifier(struct notifier_block *);

struct files_struct;
void show_fd_locks(struct seq_file *f,
                         struct file *filp, struct files_struct *files);
bool locks_owner_has_blockers(struct file_lock_context *flctx,
                        fl_owner_t owner);

static inline struct file_lock_context *
locks_inode_context(const struct inode *inode)
{
        return smp_load_acquire(&inode->i_flctx);
}

#else /* !CONFIG_FILE_LOCKING */
static inline int fcntl_getlk(struct file *file, unsigned int cmd,
                              struct flock __user *user)
{
        return -EINVAL;
}

static inline int fcntl_setlk(unsigned int fd, struct file *file,
                              unsigned int cmd, struct flock __user *user)
{
        return -EACCES;
}

#if BITS_PER_LONG == 32
static inline int fcntl_getlk64(struct file *file, unsigned int cmd,
                                struct flock64 *user)
{
        return -EINVAL;
}

static inline int fcntl_setlk64(unsigned int fd, struct file *file,
                                unsigned int cmd, struct flock64 *user)
{
        return -EACCES;
}
#endif
static inline int fcntl_setlease(unsigned int fd, struct file *filp, int arg)
{
        return -EINVAL;
}

static inline int fcntl_getlease(struct file *filp)
{
        return F_UNLCK;
}

static inline bool lock_is_unlock(struct file_lock *fl)
{
        return false;
}

static inline bool lock_is_read(struct file_lock *fl)
{
        return false;
}

static inline bool lock_is_write(struct file_lock *fl)
{
        return false;
}

static inline void locks_wake_up(struct file_lock *fl)
{
}

static inline void
locks_free_lock_context(struct inode *inode)
{
}

static inline void locks_init_lock(struct file_lock *fl)
{
        return;
}

static inline void locks_init_lease(struct file_lease *fl)
{
        return;
}

static inline void locks_copy_conflock(struct file_lock *new, struct file_lock *fl)
{
        return;
}

static inline void locks_copy_lock(struct file_lock *new, struct file_lock *fl)
{
        return;
}

static inline void locks_remove_posix(struct file *filp, fl_owner_t owner)
{
        return;
}

static inline void locks_remove_file(struct file *filp)
{
        return;
}

static inline void posix_test_lock(struct file *filp, struct file_lock *fl)
{
        return;
}

static inline int posix_lock_file(struct file *filp, struct file_lock *fl,
                                  struct file_lock *conflock)
{
        return -ENOLCK;
}

static inline int locks_delete_block(struct file_lock *waiter)
{
        return -ENOENT;
}

static inline int vfs_test_lock(struct file *filp, struct file_lock *fl)
{
        return 0;
}

static inline int vfs_lock_file(struct file *filp, unsigned int cmd,
                                struct file_lock *fl, struct file_lock *conf)
{
        return -ENOLCK;
}

static inline int vfs_cancel_lock(struct file *filp, struct file_lock *fl)
{
        return 0;
}

static inline bool vfs_inode_has_locks(struct inode *inode)
{
        return false;
}

static inline int locks_lock_inode_wait(struct inode *inode, struct file_lock *fl)
{
        return -ENOLCK;
}

static inline int __break_lease(struct inode *inode, unsigned int mode, unsigned int type)
{
        return 0;
}

static inline void lease_get_mtime(struct inode *inode,
                                   struct timespec64 *time)
{
        return;
}

static inline int generic_setlease(struct file *filp, int arg,
                                    struct file_lease **flp, void **priv)
{
        return -EINVAL;
}

static inline int kernel_setlease(struct file *filp, int arg,
                               struct file_lease **lease, void **priv)
{
        return -EINVAL;
}

static inline int vfs_setlease(struct file *filp, int arg,
                               struct file_lease **lease, void **priv)
{
        return -EINVAL;
}

static inline int lease_modify(struct file_lease *fl, int arg,
                               struct list_head *dispose)
{
        return -EINVAL;
}

struct files_struct;
static inline void show_fd_locks(struct seq_file *f,
                        struct file *filp, struct files_struct *files) {}
static inline bool locks_owner_has_blockers(struct file_lock_context *flctx,
                        fl_owner_t owner)
{
        return false;
}

static inline struct file_lock_context *
locks_inode_context(const struct inode *inode)
{
        return NULL;
}

#endif /* !CONFIG_FILE_LOCKING */

/* for walking lists of file_locks linked by fl_list */
#define for_each_file_lock(_fl, _head)        list_for_each_entry(_fl, _head, c.flc_list)

static inline int locks_lock_file_wait(struct file *filp, struct file_lock *fl)
{
        return locks_lock_inode_wait(file_inode(filp), fl);
}

#ifdef CONFIG_FILE_LOCKING
static inline int break_lease(struct inode *inode, unsigned int mode)
{
        struct file_lock_context *flctx;

        /*
         * Since this check is lockless, we must ensure that any refcounts
         * taken are done before checking i_flctx->flc_lease. Otherwise, we
         * could end up racing with tasks trying to set a new lease on this
         * file.
         */
        flctx = READ_ONCE(inode->i_flctx);
        if (!flctx)
                return 0;
        smp_mb();
        if (!list_empty_careful(&flctx->flc_lease))
                return __break_lease(inode, mode, FL_LEASE);
        return 0;
}

static inline int break_deleg(struct inode *inode, unsigned int mode)
{
        struct file_lock_context *flctx;

        /*
         * Since this check is lockless, we must ensure that any refcounts
         * taken are done before checking i_flctx->flc_lease. Otherwise, we
         * could end up racing with tasks trying to set a new lease on this
         * file.
         */
        flctx = READ_ONCE(inode->i_flctx);
        if (!flctx)
                return 0;
        smp_mb();
        if (!list_empty_careful(&flctx->flc_lease))
                return __break_lease(inode, mode, FL_DELEG);
        return 0;
}

static inline int try_break_deleg(struct inode *inode, struct inode **delegated_inode)
{
        int ret;

        ret = break_deleg(inode, O_WRONLY|O_NONBLOCK);
        if (ret == -EWOULDBLOCK && delegated_inode) {
                *delegated_inode = inode;
                ihold(inode);
        }
        return ret;
}

static inline int break_deleg_wait(struct inode **delegated_inode)
{
        int ret;

        ret = break_deleg(*delegated_inode, O_WRONLY);
        iput(*delegated_inode);
        *delegated_inode = NULL;
        return ret;
}

static inline int break_layout(struct inode *inode, bool wait)
{
        smp_mb();
        if (inode->i_flctx && !list_empty_careful(&inode->i_flctx->flc_lease))
                return __break_lease(inode,
                                wait ? O_WRONLY : O_WRONLY | O_NONBLOCK,
                                FL_LAYOUT);
        return 0;
}

#else /* !CONFIG_FILE_LOCKING */
static inline int break_lease(struct inode *inode, unsigned int mode)
{
        return 0;
}

static inline int break_deleg(struct inode *inode, unsigned int mode)
{
        return 0;
}

static inline int try_break_deleg(struct inode *inode, struct inode **delegated_inode)
{
        return 0;
}

static inline int break_deleg_wait(struct inode **delegated_inode)
{
        BUG();
        return 0;
}

static inline int break_layout(struct inode *inode, bool wait)
{
        return 0;
}

#endif /* CONFIG_FILE_LOCKING */

#endif /* _LINUX_FILELOCK_H */




























































































  251 









   46 

   46 





























    3 








    2 








  204 








  203 








  265 








  264 


























   62 








  288 



























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* delayacct.h - per-task delay accounting
 *
 * Copyright (C) Shailabh Nagar, IBM Corp. 2006
 */

#ifndef _LINUX_DELAYACCT_H
#define _LINUX_DELAYACCT_H

#include <uapi/linux/taskstats.h>

#ifdef CONFIG_TASK_DELAY_ACCT
struct task_delay_info {
        raw_spinlock_t        lock;

        /* For each stat XXX, add following, aligned appropriately
         *
         * struct timespec XXX_start, XXX_end;
         * u64 XXX_delay;
         * u32 XXX_count;
         *
         * Atomicity of updates to XXX_delay, XXX_count protected by
         * single lock above (split into XXX_lock if contention is an issue).
         */

        /*
         * XXX_count is incremented on every XXX operation, the delay
         * associated with the operation is added to XXX_delay.
         * XXX_delay contains the accumulated delay time in nanoseconds.
         */
        u64 blkio_start;
        u64 blkio_delay;        /* wait for sync block io completion */
        u64 swapin_start;
        u64 swapin_delay;        /* wait for swapin */
        u32 blkio_count;        /* total count of the number of sync block */
                                /* io operations performed */
        u32 swapin_count;        /* total count of swapin */

        u64 freepages_start;
        u64 freepages_delay;        /* wait for memory reclaim */

        u64 thrashing_start;
        u64 thrashing_delay;        /* wait for thrashing page */

        u64 compact_start;
        u64 compact_delay;        /* wait for memory compact */

        u64 wpcopy_start;
        u64 wpcopy_delay;        /* wait for write-protect copy */

        u64 irq_delay;        /* wait for IRQ/SOFTIRQ */

        u32 freepages_count;        /* total count of memory reclaim */
        u32 thrashing_count;        /* total count of thrash waits */
        u32 compact_count;        /* total count of memory compact */
        u32 wpcopy_count;        /* total count of write-protect copy */
        u32 irq_count;        /* total count of IRQ/SOFTIRQ */
};
#endif

#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/jump_label.h>

#ifdef CONFIG_TASK_DELAY_ACCT
DECLARE_STATIC_KEY_FALSE(delayacct_key);
extern int delayacct_on;        /* Delay accounting turned on/off */
extern struct kmem_cache *delayacct_cache;
extern void delayacct_init(void);

extern void __delayacct_tsk_init(struct task_struct *);
extern void __delayacct_tsk_exit(struct task_struct *);
extern void __delayacct_blkio_start(void);
extern void __delayacct_blkio_end(struct task_struct *);
extern int delayacct_add_tsk(struct taskstats *, struct task_struct *);
extern __u64 __delayacct_blkio_ticks(struct task_struct *);
extern void __delayacct_freepages_start(void);
extern void __delayacct_freepages_end(void);
extern void __delayacct_thrashing_start(bool *in_thrashing);
extern void __delayacct_thrashing_end(bool *in_thrashing);
extern void __delayacct_swapin_start(void);
extern void __delayacct_swapin_end(void);
extern void __delayacct_compact_start(void);
extern void __delayacct_compact_end(void);
extern void __delayacct_wpcopy_start(void);
extern void __delayacct_wpcopy_end(void);
extern void __delayacct_irq(struct task_struct *task, u32 delta);

static inline void delayacct_tsk_init(struct task_struct *tsk)
{
        /* reinitialize in case parent's non-null pointer was dup'ed*/
        tsk->delays = NULL;
        if (delayacct_on)
                __delayacct_tsk_init(tsk);
}

/* Free tsk->delays. Called from bad fork and __put_task_struct
 * where there's no risk of tsk->delays being accessed elsewhere
 */
static inline void delayacct_tsk_free(struct task_struct *tsk)
{
        if (tsk->delays)
                kmem_cache_free(delayacct_cache, tsk->delays);
        tsk->delays = NULL;
}

static inline void delayacct_blkio_start(void)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_blkio_start();
}

static inline void delayacct_blkio_end(struct task_struct *p)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (p->delays)
                __delayacct_blkio_end(p);
}

static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
{
        if (tsk->delays)
                return __delayacct_blkio_ticks(tsk);
        return 0;
}

static inline void delayacct_freepages_start(void)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_freepages_start();
}

static inline void delayacct_freepages_end(void)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_freepages_end();
}

static inline void delayacct_thrashing_start(bool *in_thrashing)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_thrashing_start(in_thrashing);
}

static inline void delayacct_thrashing_end(bool *in_thrashing)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_thrashing_end(in_thrashing);
}

static inline void delayacct_swapin_start(void)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_swapin_start();
}

static inline void delayacct_swapin_end(void)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_swapin_end();
}

static inline void delayacct_compact_start(void)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_compact_start();
}

static inline void delayacct_compact_end(void)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_compact_end();
}

static inline void delayacct_wpcopy_start(void)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_wpcopy_start();
}

static inline void delayacct_wpcopy_end(void)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (current->delays)
                __delayacct_wpcopy_end();
}

static inline void delayacct_irq(struct task_struct *task, u32 delta)
{
        if (!static_branch_unlikely(&delayacct_key))
                return;

        if (task->delays)
                __delayacct_irq(task, delta);
}

#else
static inline void delayacct_init(void)
{}
static inline void delayacct_tsk_init(struct task_struct *tsk)
{}
static inline void delayacct_tsk_free(struct task_struct *tsk)
{}
static inline void delayacct_blkio_start(void)
{}
static inline void delayacct_blkio_end(struct task_struct *p)
{}
static inline int delayacct_add_tsk(struct taskstats *d,
                                        struct task_struct *tsk)
{ return 0; }
static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
{ return 0; }
static inline int delayacct_is_task_waiting_on_io(struct task_struct *p)
{ return 0; }
static inline void delayacct_freepages_start(void)
{}
static inline void delayacct_freepages_end(void)
{}
static inline void delayacct_thrashing_start(bool *in_thrashing)
{}
static inline void delayacct_thrashing_end(bool *in_thrashing)
{}
static inline void delayacct_swapin_start(void)
{}
static inline void delayacct_swapin_end(void)
{}
static inline void delayacct_compact_start(void)
{}
static inline void delayacct_compact_end(void)
{}
static inline void delayacct_wpcopy_start(void)
{}
static inline void delayacct_wpcopy_end(void)
{}
static inline void delayacct_irq(struct task_struct *task, u32 delta)
{}

#endif /* CONFIG_TASK_DELAY_ACCT */

#endif









































































































































































































































































































































































































































































































































































    1 






















    1 



    1 

    1 
    1 
















    1 


    1 


    1 

































































































    1 
    1 


    1 

    1 





    1 



    1 


    1 







    1 




























































    1 



    1 



    1 


    1 
    1 





































    1 








    1 




























































































































































































































































































































    1 

    1 
























































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Scanning implementation
 *
 * Copyright 2003, Jouni Malinen <jkmaline@cc.hut.fi>
 * Copyright 2004, Instant802 Networks, Inc.
 * Copyright 2005, Devicescape Software, Inc.
 * Copyright 2006-2007        Jiri Benc <jbenc@suse.cz>
 * Copyright 2007, Michael Wu <flamingice@sourmilk.net>
 * Copyright 2013-2015  Intel Mobile Communications GmbH
 * Copyright 2016-2017  Intel Deutschland GmbH
 * Copyright (C) 2018-2024 Intel Corporation
 */

#include <linux/if_arp.h>
#include <linux/etherdevice.h>
#include <linux/rtnetlink.h>
#include <net/sch_generic.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/random.h>
#include <net/mac80211.h>

#include "ieee80211_i.h"
#include "driver-ops.h"
#include "mesh.h"

#define IEEE80211_PROBE_DELAY (HZ / 33)
#define IEEE80211_CHANNEL_TIME (HZ / 33)
#define IEEE80211_PASSIVE_CHANNEL_TIME (HZ / 9)

void ieee80211_rx_bss_put(struct ieee80211_local *local,
                          struct ieee80211_bss *bss)
{
        if (!bss)
                return;
        cfg80211_put_bss(local->hw.wiphy,
                         container_of((void *)bss, struct cfg80211_bss, priv));
}

static bool is_uapsd_supported(struct ieee802_11_elems *elems)
{
        u8 qos_info;

        if (elems->wmm_info && elems->wmm_info_len == 7
            && elems->wmm_info[5] == 1)
                qos_info = elems->wmm_info[6];
        else if (elems->wmm_param && elems->wmm_param_len == 24
                 && elems->wmm_param[5] == 1)
                qos_info = elems->wmm_param[6];
        else
                /* no valid wmm information or parameter element found */
                return false;

        return qos_info & IEEE80211_WMM_IE_AP_QOSINFO_UAPSD;
}

struct inform_bss_update_data {
        struct ieee80211_rx_status *rx_status;
        bool beacon;
};

void ieee80211_inform_bss(struct wiphy *wiphy,
                          struct cfg80211_bss *cbss,
                          const struct cfg80211_bss_ies *ies,
                          void *data)
{
        struct ieee80211_local *local = wiphy_priv(wiphy);
        struct inform_bss_update_data *update_data = data;
        struct ieee80211_bss *bss = (void *)cbss->priv;
        struct ieee80211_rx_status *rx_status;
        struct ieee802_11_elems *elems;
        int clen, srlen;

        /* This happens while joining an IBSS */
        if (!update_data)
                return;

        elems = ieee802_11_parse_elems(ies->data, ies->len, false, NULL);
        if (!elems)
                return;

        rx_status = update_data->rx_status;

        if (update_data->beacon)
                bss->device_ts_beacon = rx_status->device_timestamp;
        else
                bss->device_ts_presp = rx_status->device_timestamp;

        if (elems->parse_error) {
                if (update_data->beacon)
                        bss->corrupt_data |= IEEE80211_BSS_CORRUPT_BEACON;
                else
                        bss->corrupt_data |= IEEE80211_BSS_CORRUPT_PROBE_RESP;
        } else {
                if (update_data->beacon)
                        bss->corrupt_data &= ~IEEE80211_BSS_CORRUPT_BEACON;
                else
                        bss->corrupt_data &= ~IEEE80211_BSS_CORRUPT_PROBE_RESP;
        }

        /* save the ERP value so that it is available at association time */
        if (elems->erp_info && (!elems->parse_error ||
                                !(bss->valid_data & IEEE80211_BSS_VALID_ERP))) {
                bss->erp_value = elems->erp_info[0];
                bss->has_erp_value = true;
                if (!elems->parse_error)
                        bss->valid_data |= IEEE80211_BSS_VALID_ERP;
        }

        /* replace old supported rates if we get new values */
        if (!elems->parse_error ||
            !(bss->valid_data & IEEE80211_BSS_VALID_RATES)) {
                srlen = 0;
                if (elems->supp_rates) {
                        clen = IEEE80211_MAX_SUPP_RATES;
                        if (clen > elems->supp_rates_len)
                                clen = elems->supp_rates_len;
                        memcpy(bss->supp_rates, elems->supp_rates, clen);
                        srlen += clen;
                }
                if (elems->ext_supp_rates) {
                        clen = IEEE80211_MAX_SUPP_RATES - srlen;
                        if (clen > elems->ext_supp_rates_len)
                                clen = elems->ext_supp_rates_len;
                        memcpy(bss->supp_rates + srlen, elems->ext_supp_rates,
                               clen);
                        srlen += clen;
                }
                if (srlen) {
                        bss->supp_rates_len = srlen;
                        if (!elems->parse_error)
                                bss->valid_data |= IEEE80211_BSS_VALID_RATES;
                }
        }

        if (!elems->parse_error ||
            !(bss->valid_data & IEEE80211_BSS_VALID_WMM)) {
                bss->wmm_used = elems->wmm_param || elems->wmm_info;
                bss->uapsd_supported = is_uapsd_supported(elems);
                if (!elems->parse_error)
                        bss->valid_data |= IEEE80211_BSS_VALID_WMM;
        }

        if (update_data->beacon) {
                struct ieee80211_supported_band *sband =
                        local->hw.wiphy->bands[rx_status->band];
                if (!(rx_status->encoding == RX_ENC_HT) &&
                    !(rx_status->encoding == RX_ENC_VHT))
                        bss->beacon_rate =
                                &sband->bitrates[rx_status->rate_idx];
        }

        if (elems->vht_cap_elem)
                bss->vht_cap_info =
                        le32_to_cpu(elems->vht_cap_elem->vht_cap_info);
        else
                bss->vht_cap_info = 0;

        kfree(elems);
}

struct ieee80211_bss *
ieee80211_bss_info_update(struct ieee80211_local *local,
                          struct ieee80211_rx_status *rx_status,
                          struct ieee80211_mgmt *mgmt, size_t len,
                          struct ieee80211_channel *channel)
{
        bool beacon = ieee80211_is_beacon(mgmt->frame_control) ||
                      ieee80211_is_s1g_beacon(mgmt->frame_control);
        struct cfg80211_bss *cbss;
        struct inform_bss_update_data update_data = {
                .rx_status = rx_status,
                .beacon = beacon,
        };
        struct cfg80211_inform_bss bss_meta = {
                .boottime_ns = rx_status->boottime_ns,
                .drv_data = (void *)&update_data,
        };
        bool signal_valid;
        struct ieee80211_sub_if_data *scan_sdata;

        if (rx_status->flag & RX_FLAG_NO_SIGNAL_VAL)
                bss_meta.signal = 0; /* invalid signal indication */
        else if (ieee80211_hw_check(&local->hw, SIGNAL_DBM))
                bss_meta.signal = rx_status->signal * 100;
        else if (ieee80211_hw_check(&local->hw, SIGNAL_UNSPEC))
                bss_meta.signal = (rx_status->signal * 100) / local->hw.max_signal;

        bss_meta.chan = channel;

        rcu_read_lock();
        scan_sdata = rcu_dereference(local->scan_sdata);
        if (scan_sdata && scan_sdata->vif.type == NL80211_IFTYPE_STATION &&
            scan_sdata->vif.cfg.assoc &&
            ieee80211_have_rx_timestamp(rx_status)) {
                struct ieee80211_bss_conf *link_conf = NULL;

                /* for an MLO connection, set the TSF data only in case we have
                 * an indication on which of the links the frame was received
                 */
                if (ieee80211_vif_is_mld(&scan_sdata->vif)) {
                        if (rx_status->link_valid) {
                                s8 link_id = rx_status->link_id;

                                link_conf =
                                        rcu_dereference(scan_sdata->vif.link_conf[link_id]);
                        }
                } else {
                        link_conf = &scan_sdata->vif.bss_conf;
                }

                if (link_conf) {
                        bss_meta.parent_tsf =
                                ieee80211_calculate_rx_timestamp(local,
                                                                 rx_status,
                                                                 len + FCS_LEN,
                                                                 24);

                        ether_addr_copy(bss_meta.parent_bssid,
                                        link_conf->bssid);
                }
        }
        rcu_read_unlock();

        cbss = cfg80211_inform_bss_frame_data(local->hw.wiphy, &bss_meta,
                                              mgmt, len, GFP_ATOMIC);
        if (!cbss)
                return NULL;

        /* In case the signal is invalid update the status */
        signal_valid = channel == cbss->channel;
        if (!signal_valid)
                rx_status->flag |= RX_FLAG_NO_SIGNAL_VAL;

        return (void *)cbss->priv;
}

static bool ieee80211_scan_accept_presp(struct ieee80211_sub_if_data *sdata,
                                        struct ieee80211_channel *channel,
                                        u32 scan_flags, const u8 *da)
{
        if (!sdata)
                return false;

        /* accept broadcast on 6 GHz and for OCE */
        if (is_broadcast_ether_addr(da) &&
            (channel->band == NL80211_BAND_6GHZ ||
             scan_flags & NL80211_SCAN_FLAG_ACCEPT_BCAST_PROBE_RESP))
                return true;

        if (scan_flags & NL80211_SCAN_FLAG_RANDOM_ADDR)
                return true;
        return ether_addr_equal(da, sdata->vif.addr);
}

void ieee80211_scan_rx(struct ieee80211_local *local, struct sk_buff *skb)
{
        struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
        struct ieee80211_mgmt *mgmt = (void *)skb->data;
        struct ieee80211_bss *bss;
        struct ieee80211_channel *channel;
        size_t min_hdr_len = offsetof(struct ieee80211_mgmt,
                                      u.probe_resp.variable);

        if (!ieee80211_is_probe_resp(mgmt->frame_control) &&
            !ieee80211_is_beacon(mgmt->frame_control) &&
            !ieee80211_is_s1g_beacon(mgmt->frame_control))
                return;

        if (ieee80211_is_s1g_beacon(mgmt->frame_control)) {
                if (ieee80211_is_s1g_short_beacon(mgmt->frame_control))
                        min_hdr_len = offsetof(struct ieee80211_ext,
                                               u.s1g_short_beacon.variable);
                else
                        min_hdr_len = offsetof(struct ieee80211_ext,
                                               u.s1g_beacon);
        }

        if (skb->len < min_hdr_len)
                return;

        if (test_and_clear_bit(SCAN_BEACON_WAIT, &local->scanning)) {
                /*
                 * we were passive scanning because of radar/no-IR, but
                 * the beacon/proberesp rx gives us an opportunity to upgrade
                 * to active scan
                 */
                set_bit(SCAN_BEACON_DONE, &local->scanning);
                wiphy_delayed_work_queue(local->hw.wiphy, &local->scan_work, 0);
        }

        channel = ieee80211_get_channel_khz(local->hw.wiphy,
                                            ieee80211_rx_status_to_khz(rx_status));

        if (!channel || channel->flags & IEEE80211_CHAN_DISABLED)
                return;

        if (ieee80211_is_probe_resp(mgmt->frame_control)) {
                struct ieee80211_sub_if_data *sdata1, *sdata2;
                struct cfg80211_scan_request *scan_req;
                struct cfg80211_sched_scan_request *sched_scan_req;
                u32 scan_req_flags = 0, sched_scan_req_flags = 0;

                sdata1 = rcu_dereference(local->scan_sdata);
                sdata2 = rcu_dereference(local->sched_scan_sdata);

                if (likely(!sdata1 && !sdata2))
                        return;

                scan_req = rcu_dereference(local->scan_req);
                sched_scan_req = rcu_dereference(local->sched_scan_req);

                if (scan_req)
                        scan_req_flags = scan_req->flags;

                if (sched_scan_req)
                        sched_scan_req_flags = sched_scan_req->flags;

                /* ignore ProbeResp to foreign address or non-bcast (OCE)
                 * unless scanning with randomised address
                 */
                if (!ieee80211_scan_accept_presp(sdata1, channel,
                                                 scan_req_flags,
                                                 mgmt->da) &&
                    !ieee80211_scan_accept_presp(sdata2, channel,
                                                 sched_scan_req_flags,
                                                 mgmt->da))
                        return;
        } else {
                /* Beacons are expected only with broadcast address */
                if (!is_broadcast_ether_addr(mgmt->da))
                        return;
        }

        /* Do not update the BSS table in case of only monitor interfaces */
        if (local->open_count == local->monitors)
                return;

        bss = ieee80211_bss_info_update(local, rx_status,
                                        mgmt, skb->len,
                                        channel);
        if (bss)
                ieee80211_rx_bss_put(local, bss);
}

static void ieee80211_prepare_scan_chandef(struct cfg80211_chan_def *chandef)
{
        memset(chandef, 0, sizeof(*chandef));

        chandef->width = NL80211_CHAN_WIDTH_20_NOHT;
}

/* return false if no more work */
static bool ieee80211_prep_hw_scan(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct cfg80211_scan_request *req;
        struct cfg80211_chan_def chandef;
        u8 bands_used = 0;
        int i, ielen;
        u32 *n_chans;
        u32 flags = 0;

        req = rcu_dereference_protected(local->scan_req,
                                        lockdep_is_held(&local->hw.wiphy->mtx));

        if (test_bit(SCAN_HW_CANCELLED, &local->scanning))
                return false;

        if (ieee80211_hw_check(&local->hw, SINGLE_SCAN_ON_ALL_BANDS)) {
                local->hw_scan_req->req.n_channels = req->n_channels;

                for (i = 0; i < req->n_channels; i++) {
                        local->hw_scan_req->req.channels[i] = req->channels[i];
                        bands_used |= BIT(req->channels[i]->band);
                }
        } else {
                do {
                        if (local->hw_scan_band == NUM_NL80211_BANDS)
                                return false;

                        n_chans = &local->hw_scan_req->req.n_channels;
                        *n_chans = 0;

                        for (i = 0; i < req->n_channels; i++) {
                                if (req->channels[i]->band !=
                                    local->hw_scan_band)
                                        continue;
                                local->hw_scan_req->req.channels[(*n_chans)++] =
                                                        req->channels[i];

                                bands_used |= BIT(req->channels[i]->band);
                        }

                        local->hw_scan_band++;
                } while (!*n_chans);
        }

        ieee80211_prepare_scan_chandef(&chandef);

        if (req->flags & NL80211_SCAN_FLAG_MIN_PREQ_CONTENT)
                flags |= IEEE80211_PROBE_FLAG_MIN_CONTENT;

        ielen = ieee80211_build_preq_ies(sdata,
                                         (u8 *)local->hw_scan_req->req.ie,
                                         local->hw_scan_ies_bufsize,
                                         &local->hw_scan_req->ies,
                                         req->ie, req->ie_len,
                                         bands_used, req->rates, &chandef,
                                         flags);
        if (ielen < 0)
                return false;
        local->hw_scan_req->req.ie_len = ielen;
        local->hw_scan_req->req.no_cck = req->no_cck;
        ether_addr_copy(local->hw_scan_req->req.mac_addr, req->mac_addr);
        ether_addr_copy(local->hw_scan_req->req.mac_addr_mask,
                        req->mac_addr_mask);
        ether_addr_copy(local->hw_scan_req->req.bssid, req->bssid);

        return true;
}

static void __ieee80211_scan_completed(struct ieee80211_hw *hw, bool aborted)
{
        struct ieee80211_local *local = hw_to_local(hw);
        bool hw_scan = test_bit(SCAN_HW_SCANNING, &local->scanning);
        bool was_scanning = local->scanning;
        struct cfg80211_scan_request *scan_req;
        struct ieee80211_sub_if_data *scan_sdata;
        struct ieee80211_sub_if_data *sdata;

        lockdep_assert_wiphy(local->hw.wiphy);

        /*
         * It's ok to abort a not-yet-running scan (that
         * we have one at all will be verified by checking
         * local->scan_req next), but not to complete it
         * successfully.
         */
        if (WARN_ON(!local->scanning && !aborted))
                aborted = true;

        if (WARN_ON(!local->scan_req))
                return;

        scan_sdata = rcu_dereference_protected(local->scan_sdata,
                                               lockdep_is_held(&local->hw.wiphy->mtx));

        if (hw_scan && !aborted &&
            !ieee80211_hw_check(&local->hw, SINGLE_SCAN_ON_ALL_BANDS) &&
            ieee80211_prep_hw_scan(scan_sdata)) {
                int rc;

                rc = drv_hw_scan(local,
                        rcu_dereference_protected(local->scan_sdata,
                                                  lockdep_is_held(&local->hw.wiphy->mtx)),
                        local->hw_scan_req);

                if (rc == 0)
                        return;

                /* HW scan failed and is going to be reported as aborted,
                 * so clear old scan info.
                 */
                memset(&local->scan_info, 0, sizeof(local->scan_info));
                aborted = true;
        }

        kfree(local->hw_scan_req);
        local->hw_scan_req = NULL;

        scan_req = rcu_dereference_protected(local->scan_req,
                                             lockdep_is_held(&local->hw.wiphy->mtx));

        RCU_INIT_POINTER(local->scan_req, NULL);
        RCU_INIT_POINTER(local->scan_sdata, NULL);

        local->scanning = 0;
        local->scan_chandef.chan = NULL;

        synchronize_rcu();

        if (scan_req != local->int_scan_req) {
                local->scan_info.aborted = aborted;
                cfg80211_scan_done(scan_req, &local->scan_info);
        }

        /* Set power back to normal operating levels. */
        ieee80211_hw_conf_chan(local);

        if (!hw_scan && was_scanning) {
                ieee80211_configure_filter(local);
                drv_sw_scan_complete(local, scan_sdata);
                ieee80211_offchannel_return(local);
        }

        ieee80211_recalc_idle(local);

        ieee80211_mlme_notify_scan_completed(local);
        ieee80211_ibss_notify_scan_completed(local);

        /* Requeue all the work that might have been ignored while
         * the scan was in progress; if there was none this will
         * just be a no-op for the particular interface.
         */
        list_for_each_entry(sdata, &local->interfaces, list) {
                if (ieee80211_sdata_running(sdata))
                        wiphy_work_queue(sdata->local->hw.wiphy, &sdata->work);
        }

        if (was_scanning)
                ieee80211_start_next_roc(local);
}

void ieee80211_scan_completed(struct ieee80211_hw *hw,
                              struct cfg80211_scan_info *info)
{
        struct ieee80211_local *local = hw_to_local(hw);

        trace_api_scan_completed(local, info->aborted);

        set_bit(SCAN_COMPLETED, &local->scanning);
        if (info->aborted)
                set_bit(SCAN_ABORTED, &local->scanning);

        memcpy(&local->scan_info, info, sizeof(*info));

        wiphy_delayed_work_queue(local->hw.wiphy, &local->scan_work, 0);
}
EXPORT_SYMBOL(ieee80211_scan_completed);

static int ieee80211_start_sw_scan(struct ieee80211_local *local,
                                   struct ieee80211_sub_if_data *sdata)
{
        /* Software scan is not supported in multi-channel cases */
        if (!local->emulate_chanctx)
                return -EOPNOTSUPP;

        /*
         * Hardware/driver doesn't support hw_scan, so use software
         * scanning instead. First send a nullfunc frame with power save
         * bit on so that AP will buffer the frames for us while we are not
         * listening, then send probe requests to each channel and wait for
         * the responses. After all channels are scanned, tune back to the
         * original channel and send a nullfunc frame with power save bit
         * off to trigger the AP to send us all the buffered frames.
         *
         * Note that while local->sw_scanning is true everything else but
         * nullfunc frames and probe requests will be dropped in
         * ieee80211_tx_h_check_assoc().
         */
        drv_sw_scan_start(local, sdata, local->scan_addr);

        local->leave_oper_channel_time = jiffies;
        local->next_scan_state = SCAN_DECISION;
        local->scan_channel_idx = 0;

        ieee80211_offchannel_stop_vifs(local);

        /* ensure nullfunc is transmitted before leaving operating channel */
        ieee80211_flush_queues(local, NULL, false);

        ieee80211_configure_filter(local);

        /* We need to set power level at maximum rate for scanning. */
        ieee80211_hw_conf_chan(local);

        wiphy_delayed_work_queue(local->hw.wiphy, &local->scan_work, 0);

        return 0;
}

static bool __ieee80211_can_leave_ch(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_sub_if_data *sdata_iter;
        unsigned int link_id;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (!ieee80211_is_radar_required(local))
                return true;

        if (!regulatory_pre_cac_allowed(local->hw.wiphy))
                return false;

        list_for_each_entry(sdata_iter, &local->interfaces, list) {
                for_each_valid_link(&sdata_iter->wdev, link_id)
                        if (sdata_iter->wdev.links[link_id].cac_started)
                                return false;
        }

        return true;
}

static bool ieee80211_can_scan(struct ieee80211_local *local,
                               struct ieee80211_sub_if_data *sdata)
{
        if (!__ieee80211_can_leave_ch(sdata))
                return false;

        if (!list_empty(&local->roc_list))
                return false;

        if (sdata->vif.type == NL80211_IFTYPE_STATION &&
            sdata->u.mgd.flags & IEEE80211_STA_CONNECTION_POLL)
                return false;

        return true;
}

void ieee80211_run_deferred_scan(struct ieee80211_local *local)
{
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!local->scan_req || local->scanning)
                return;

        if (!ieee80211_can_scan(local,
                                rcu_dereference_protected(
                                        local->scan_sdata,
                                        lockdep_is_held(&local->hw.wiphy->mtx))))
                return;

        wiphy_delayed_work_queue(local->hw.wiphy, &local->scan_work,
                                 round_jiffies_relative(0));
}

static void ieee80211_send_scan_probe_req(struct ieee80211_sub_if_data *sdata,
                                          const u8 *src, const u8 *dst,
                                          const u8 *ssid, size_t ssid_len,
                                          const u8 *ie, size_t ie_len,
                                          u32 ratemask, u32 flags, u32 tx_flags,
                                          struct ieee80211_channel *channel)
{
        struct sk_buff *skb;

        skb = ieee80211_build_probe_req(sdata, src, dst, ratemask, channel,
                                        ssid, ssid_len,
                                        ie, ie_len, flags);

        if (skb) {
                if (flags & IEEE80211_PROBE_FLAG_RANDOM_SN) {
                        struct ieee80211_hdr *hdr = (void *)skb->data;
                        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
                        u16 sn = get_random_u16();

                        info->control.flags |= IEEE80211_TX_CTRL_NO_SEQNO;
                        hdr->seq_ctrl =
                                cpu_to_le16(IEEE80211_SN_TO_SEQ(sn));
                }
                IEEE80211_SKB_CB(skb)->flags |= tx_flags;
                IEEE80211_SKB_CB(skb)->control.flags |= IEEE80211_TX_CTRL_DONT_USE_RATE_MASK;
                ieee80211_tx_skb_tid_band(sdata, skb, 7, channel->band);
        }
}

static void ieee80211_scan_state_send_probe(struct ieee80211_local *local,
                                            unsigned long *next_delay)
{
        int i;
        struct ieee80211_sub_if_data *sdata;
        struct cfg80211_scan_request *scan_req;
        enum nl80211_band band = local->hw.conf.chandef.chan->band;
        u32 flags = 0, tx_flags;

        scan_req = rcu_dereference_protected(local->scan_req,
                                             lockdep_is_held(&local->hw.wiphy->mtx));

        tx_flags = IEEE80211_TX_INTFL_OFFCHAN_TX_OK;
        if (scan_req->no_cck)
                tx_flags |= IEEE80211_TX_CTL_NO_CCK_RATE;
        if (scan_req->flags & NL80211_SCAN_FLAG_MIN_PREQ_CONTENT)
                flags |= IEEE80211_PROBE_FLAG_MIN_CONTENT;
        if (scan_req->flags & NL80211_SCAN_FLAG_RANDOM_SN)
                flags |= IEEE80211_PROBE_FLAG_RANDOM_SN;

        sdata = rcu_dereference_protected(local->scan_sdata,
                                          lockdep_is_held(&local->hw.wiphy->mtx));

        for (i = 0; i < scan_req->n_ssids; i++)
                ieee80211_send_scan_probe_req(
                        sdata, local->scan_addr, scan_req->bssid,
                        scan_req->ssids[i].ssid, scan_req->ssids[i].ssid_len,
                        scan_req->ie, scan_req->ie_len,
                        scan_req->rates[band], flags,
                        tx_flags, local->hw.conf.chandef.chan);

        /*
         * After sending probe requests, wait for probe responses
         * on the channel.
         */
        *next_delay = msecs_to_jiffies(scan_req->duration) >
                      IEEE80211_PROBE_DELAY + IEEE80211_CHANNEL_TIME ?
                      msecs_to_jiffies(scan_req->duration) - IEEE80211_PROBE_DELAY :
                      IEEE80211_CHANNEL_TIME;
        local->next_scan_state = SCAN_DECISION;
}

static int __ieee80211_start_scan(struct ieee80211_sub_if_data *sdata,
                                  struct cfg80211_scan_request *req)
{
        struct ieee80211_local *local = sdata->local;
        bool hw_scan = local->ops->hw_scan;
        int rc;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (local->scan_req)
                return -EBUSY;

        /* For an MLO connection, if a link ID was specified, validate that it
         * is indeed active.
         */
        if (ieee80211_vif_is_mld(&sdata->vif) && req->tsf_report_link_id >= 0 &&
            !(sdata->vif.active_links & BIT(req->tsf_report_link_id)))
                return -EINVAL;

        if (!__ieee80211_can_leave_ch(sdata))
                return -EBUSY;

        if (!ieee80211_can_scan(local, sdata)) {
                /* wait for the work to finish/time out */
                rcu_assign_pointer(local->scan_req, req);
                rcu_assign_pointer(local->scan_sdata, sdata);
                return 0;
        }

 again:
        if (hw_scan) {
                u8 *ies;

                local->hw_scan_ies_bufsize = local->scan_ies_len + req->ie_len;

                if (ieee80211_hw_check(&local->hw, SINGLE_SCAN_ON_ALL_BANDS)) {
                        int i, n_bands = 0;
                        u8 bands_counted = 0;

                        for (i = 0; i < req->n_channels; i++) {
                                if (bands_counted & BIT(req->channels[i]->band))
                                        continue;
                                bands_counted |= BIT(req->channels[i]->band);
                                n_bands++;
                        }

                        local->hw_scan_ies_bufsize *= n_bands;
                }

                local->hw_scan_req = kmalloc(struct_size(local->hw_scan_req,
                                                         req.channels,
                                                         req->n_channels) +
                                             local->hw_scan_ies_bufsize,
                                             GFP_KERNEL);
                if (!local->hw_scan_req)
                        return -ENOMEM;

                local->hw_scan_req->req.ssids = req->ssids;
                local->hw_scan_req->req.n_ssids = req->n_ssids;
                /* None of the channels are actually set
                 * up but let UBSAN know the boundaries.
                 */
                local->hw_scan_req->req.n_channels = req->n_channels;

                ies = (u8 *)local->hw_scan_req +
                        sizeof(*local->hw_scan_req) +
                        req->n_channels * sizeof(req->channels[0]);
                local->hw_scan_req->req.ie = ies;
                local->hw_scan_req->req.flags = req->flags;
                eth_broadcast_addr(local->hw_scan_req->req.bssid);
                local->hw_scan_req->req.duration = req->duration;
                local->hw_scan_req->req.duration_mandatory =
                        req->duration_mandatory;
                local->hw_scan_req->req.tsf_report_link_id =
                        req->tsf_report_link_id;

                local->hw_scan_band = 0;
                local->hw_scan_req->req.n_6ghz_params = req->n_6ghz_params;
                local->hw_scan_req->req.scan_6ghz_params =
                        req->scan_6ghz_params;
                local->hw_scan_req->req.scan_6ghz = req->scan_6ghz;

                /*
                 * After allocating local->hw_scan_req, we must
                 * go through until ieee80211_prep_hw_scan(), so
                 * anything that might be changed here and leave
                 * this function early must not go after this
                 * allocation.
                 */
        }

        rcu_assign_pointer(local->scan_req, req);
        rcu_assign_pointer(local->scan_sdata, sdata);

        if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR)
                get_random_mask_addr(local->scan_addr,
                                     req->mac_addr,
                                     req->mac_addr_mask);
        else
                memcpy(local->scan_addr, sdata->vif.addr, ETH_ALEN);

        if (hw_scan) {
                __set_bit(SCAN_HW_SCANNING, &local->scanning);
        } else if ((req->n_channels == 1) &&
                   (req->channels[0] == local->hw.conf.chandef.chan)) {
                /*
                 * If we are scanning only on the operating channel
                 * then we do not need to stop normal activities
                 */
                unsigned long next_delay;

                __set_bit(SCAN_ONCHANNEL_SCANNING, &local->scanning);

                ieee80211_recalc_idle(local);

                /* Notify driver scan is starting, keep order of operations
                 * same as normal software scan, in case that matters. */
                drv_sw_scan_start(local, sdata, local->scan_addr);

                ieee80211_configure_filter(local); /* accept probe-responses */

                /* We need to ensure power level is at max for scanning. */
                ieee80211_hw_conf_chan(local);

                if ((req->channels[0]->flags & (IEEE80211_CHAN_NO_IR |
                                                IEEE80211_CHAN_RADAR)) ||
                    !req->n_ssids) {
                        next_delay = IEEE80211_PASSIVE_CHANNEL_TIME;
                        if (req->n_ssids)
                                set_bit(SCAN_BEACON_WAIT, &local->scanning);
                } else {
                        ieee80211_scan_state_send_probe(local, &next_delay);
                        next_delay = IEEE80211_CHANNEL_TIME;
                }

                /* Now, just wait a bit and we are all done! */
                wiphy_delayed_work_queue(local->hw.wiphy, &local->scan_work,
                                         next_delay);
                return 0;
        } else {
                /* Do normal software scan */
                __set_bit(SCAN_SW_SCANNING, &local->scanning);
        }

        ieee80211_recalc_idle(local);

        if (hw_scan) {
                WARN_ON(!ieee80211_prep_hw_scan(sdata));
                rc = drv_hw_scan(local, sdata, local->hw_scan_req);
        } else {
                rc = ieee80211_start_sw_scan(local, sdata);
        }

        if (rc) {
                kfree(local->hw_scan_req);
                local->hw_scan_req = NULL;
                local->scanning = 0;

                ieee80211_recalc_idle(local);

                local->scan_req = NULL;
                RCU_INIT_POINTER(local->scan_sdata, NULL);
        }

        if (hw_scan && rc == 1) {
                /*
                 * we can't fall back to software for P2P-GO
                 * as it must update NoA etc.
                 */
                if (ieee80211_vif_type_p2p(&sdata->vif) ==
                                NL80211_IFTYPE_P2P_GO)
                        return -EOPNOTSUPP;
                hw_scan = false;
                goto again;
        }

        return rc;
}

static unsigned long
ieee80211_scan_get_channel_time(struct ieee80211_channel *chan)
{
        /*
         * TODO: channel switching also consumes quite some time,
         * add that delay as well to get a better estimation
         */
        if (chan->flags & (IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR))
                return IEEE80211_PASSIVE_CHANNEL_TIME;
        return IEEE80211_PROBE_DELAY + IEEE80211_CHANNEL_TIME;
}

static void ieee80211_scan_state_decision(struct ieee80211_local *local,
                                          unsigned long *next_delay)
{
        bool associated = false;
        bool tx_empty = true;
        bool bad_latency;
        struct ieee80211_sub_if_data *sdata;
        struct ieee80211_channel *next_chan;
        enum mac80211_scan_state next_scan_state;
        struct cfg80211_scan_request *scan_req;

        lockdep_assert_wiphy(local->hw.wiphy);

        /*
         * check if at least one STA interface is associated,
         * check if at least one STA interface has pending tx frames
         * and grab the lowest used beacon interval
         */
        list_for_each_entry(sdata, &local->interfaces, list) {
                if (!ieee80211_sdata_running(sdata))
                        continue;

                if (sdata->vif.type == NL80211_IFTYPE_STATION) {
                        if (sdata->u.mgd.associated) {
                                associated = true;

                                if (!qdisc_all_tx_empty(sdata->dev)) {
                                        tx_empty = false;
                                        break;
                                }
                        }
                }
        }

        scan_req = rcu_dereference_protected(local->scan_req,
                                             lockdep_is_held(&local->hw.wiphy->mtx));

        next_chan = scan_req->channels[local->scan_channel_idx];

        /*
         * we're currently scanning a different channel, let's
         * see if we can scan another channel without interfering
         * with the current traffic situation.
         *
         * Keep good latency, do not stay off-channel more than 125 ms.
         */

        bad_latency = time_after(jiffies +
                                 ieee80211_scan_get_channel_time(next_chan),
                                 local->leave_oper_channel_time + HZ / 8);

        if (associated && !tx_empty) {
                if (scan_req->flags & NL80211_SCAN_FLAG_LOW_PRIORITY)
                        next_scan_state = SCAN_ABORT;
                else
                        next_scan_state = SCAN_SUSPEND;
        } else if (associated && bad_latency) {
                next_scan_state = SCAN_SUSPEND;
        } else {
                next_scan_state = SCAN_SET_CHANNEL;
        }

        local->next_scan_state = next_scan_state;

        *next_delay = 0;
}

static void ieee80211_scan_state_set_channel(struct ieee80211_local *local,
                                             unsigned long *next_delay)
{
        int skip;
        struct ieee80211_channel *chan;
        struct cfg80211_scan_request *scan_req;

        scan_req = rcu_dereference_protected(local->scan_req,
                                             lockdep_is_held(&local->hw.wiphy->mtx));

        skip = 0;
        chan = scan_req->channels[local->scan_channel_idx];

        local->scan_chandef.chan = chan;
        local->scan_chandef.center_freq1 = chan->center_freq;
        local->scan_chandef.freq1_offset = chan->freq_offset;
        local->scan_chandef.center_freq2 = 0;

        /* For scanning on the S1G band, detect the channel width according to
         * the channel being scanned.
         */
        if (chan->band == NL80211_BAND_S1GHZ) {
                local->scan_chandef.width = ieee80211_s1g_channel_width(chan);
                goto set_channel;
        }

        /* If scanning on oper channel, use whatever channel-type
         * is currently in use.
         */
        if (chan == local->hw.conf.chandef.chan)
                local->scan_chandef = local->hw.conf.chandef;
        else
                local->scan_chandef.width = NL80211_CHAN_WIDTH_20_NOHT;

set_channel:
        if (ieee80211_hw_conf_chan(local))
                skip = 1;

        /* advance state machine to next channel/band */
        local->scan_channel_idx++;

        if (skip) {
                /* if we skip this channel return to the decision state */
                local->next_scan_state = SCAN_DECISION;
                return;
        }

        /*
         * Probe delay is used to update the NAV, cf. 11.1.3.2.2
         * (which unfortunately doesn't say _why_ step a) is done,
         * but it waits for the probe delay or until a frame is
         * received - and the received frame would update the NAV).
         * For now, we do not support waiting until a frame is
         * received.
         *
         * In any case, it is not necessary for a passive scan.
         */
        if ((chan->flags & (IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR)) ||
            !scan_req->n_ssids) {
                *next_delay = max(msecs_to_jiffies(scan_req->duration),
                                  IEEE80211_PASSIVE_CHANNEL_TIME);
                local->next_scan_state = SCAN_DECISION;
                if (scan_req->n_ssids)
                        set_bit(SCAN_BEACON_WAIT, &local->scanning);
                return;
        }

        /* active scan, send probes */
        *next_delay = IEEE80211_PROBE_DELAY;
        local->next_scan_state = SCAN_SEND_PROBE;
}

static void ieee80211_scan_state_suspend(struct ieee80211_local *local,
                                         unsigned long *next_delay)
{
        /* switch back to the operating channel */
        local->scan_chandef.chan = NULL;
        ieee80211_hw_conf_chan(local);

        /* disable PS */
        ieee80211_offchannel_return(local);

        *next_delay = HZ / 5;
        /* afterwards, resume scan & go to next channel */
        local->next_scan_state = SCAN_RESUME;
}

static void ieee80211_scan_state_resume(struct ieee80211_local *local,
                                        unsigned long *next_delay)
{
        ieee80211_offchannel_stop_vifs(local);

        if (local->ops->flush) {
                ieee80211_flush_queues(local, NULL, false);
                *next_delay = 0;
        } else
                *next_delay = HZ / 10;

        /* remember when we left the operating channel */
        local->leave_oper_channel_time = jiffies;

        /* advance to the next channel to be scanned */
        local->next_scan_state = SCAN_SET_CHANNEL;
}

void ieee80211_scan_work(struct wiphy *wiphy, struct wiphy_work *work)
{
        struct ieee80211_local *local =
                container_of(work, struct ieee80211_local, scan_work.work);
        struct ieee80211_sub_if_data *sdata;
        struct cfg80211_scan_request *scan_req;
        unsigned long next_delay = 0;
        bool aborted;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (!ieee80211_can_run_worker(local)) {
                aborted = true;
                goto out_complete;
        }

        sdata = rcu_dereference_protected(local->scan_sdata,
                                          lockdep_is_held(&local->hw.wiphy->mtx));
        scan_req = rcu_dereference_protected(local->scan_req,
                                             lockdep_is_held(&local->hw.wiphy->mtx));

        /* When scanning on-channel, the first-callback means completed. */
        if (test_bit(SCAN_ONCHANNEL_SCANNING, &local->scanning)) {
                aborted = test_and_clear_bit(SCAN_ABORTED, &local->scanning);
                goto out_complete;
        }

        if (test_and_clear_bit(SCAN_COMPLETED, &local->scanning)) {
                aborted = test_and_clear_bit(SCAN_ABORTED, &local->scanning);
                goto out_complete;
        }

        if (!sdata || !scan_req)
                return;

        if (!local->scanning) {
                int rc;

                RCU_INIT_POINTER(local->scan_req, NULL);
                RCU_INIT_POINTER(local->scan_sdata, NULL);

                rc = __ieee80211_start_scan(sdata, scan_req);
                if (!rc)
                        return;
                /* need to complete scan in cfg80211 */
                rcu_assign_pointer(local->scan_req, scan_req);
                aborted = true;
                goto out_complete;
        }

        clear_bit(SCAN_BEACON_WAIT, &local->scanning);

        /*
         * as long as no delay is required advance immediately
         * without scheduling a new work
         */
        do {
                if (!ieee80211_sdata_running(sdata)) {
                        aborted = true;
                        goto out_complete;
                }

                if (test_and_clear_bit(SCAN_BEACON_DONE, &local->scanning) &&
                    local->next_scan_state == SCAN_DECISION)
                        local->next_scan_state = SCAN_SEND_PROBE;

                switch (local->next_scan_state) {
                case SCAN_DECISION:
                        /* if no more bands/channels left, complete scan */
                        if (local->scan_channel_idx >= scan_req->n_channels) {
                                aborted = false;
                                goto out_complete;
                        }
                        ieee80211_scan_state_decision(local, &next_delay);
                        break;
                case SCAN_SET_CHANNEL:
                        ieee80211_scan_state_set_channel(local, &next_delay);
                        break;
                case SCAN_SEND_PROBE:
                        ieee80211_scan_state_send_probe(local, &next_delay);
                        break;
                case SCAN_SUSPEND:
                        ieee80211_scan_state_suspend(local, &next_delay);
                        break;
                case SCAN_RESUME:
                        ieee80211_scan_state_resume(local, &next_delay);
                        break;
                case SCAN_ABORT:
                        aborted = true;
                        goto out_complete;
                }
        } while (next_delay == 0);

        wiphy_delayed_work_queue(local->hw.wiphy, &local->scan_work,
                                 next_delay);
        return;

out_complete:
        __ieee80211_scan_completed(&local->hw, aborted);
}

int ieee80211_request_scan(struct ieee80211_sub_if_data *sdata,
                           struct cfg80211_scan_request *req)
{
        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        return __ieee80211_start_scan(sdata, req);
}

int ieee80211_request_ibss_scan(struct ieee80211_sub_if_data *sdata,
                                const u8 *ssid, u8 ssid_len,
                                struct ieee80211_channel **channels,
                                unsigned int n_channels)
{
        struct ieee80211_local *local = sdata->local;
        int i, n_ch = 0;
        enum nl80211_band band;

        lockdep_assert_wiphy(local->hw.wiphy);

        /* busy scanning */
        if (local->scan_req)
                return -EBUSY;

        /* fill internal scan request */
        if (!channels) {
                int max_n;

                for (band = 0; band < NUM_NL80211_BANDS; band++) {
                        if (!local->hw.wiphy->bands[band] ||
                            band == NL80211_BAND_6GHZ)
                                continue;

                        max_n = local->hw.wiphy->bands[band]->n_channels;
                        for (i = 0; i < max_n; i++) {
                                struct ieee80211_channel *tmp_ch =
                                    &local->hw.wiphy->bands[band]->channels[i];

                                if (tmp_ch->flags & (IEEE80211_CHAN_NO_IR |
                                                     IEEE80211_CHAN_DISABLED) ||
                                    !cfg80211_wdev_channel_allowed(&sdata->wdev,
                                                                   tmp_ch))
                                        continue;

                                local->int_scan_req->channels[n_ch] = tmp_ch;
                                n_ch++;
                        }
                }

                if (WARN_ON_ONCE(n_ch == 0))
                        return -EINVAL;

                local->int_scan_req->n_channels = n_ch;
        } else {
                for (i = 0; i < n_channels; i++) {
                        if (channels[i]->flags & (IEEE80211_CHAN_NO_IR |
                                                  IEEE80211_CHAN_DISABLED) ||
                            !cfg80211_wdev_channel_allowed(&sdata->wdev,
                                                           channels[i]))
                                continue;

                        local->int_scan_req->channels[n_ch] = channels[i];
                        n_ch++;
                }

                if (n_ch == 0)
                        return -EINVAL;

                local->int_scan_req->n_channels = n_ch;
        }

        local->int_scan_req->ssids = &local->scan_ssid;
        local->int_scan_req->n_ssids = 1;
        memcpy(local->int_scan_req->ssids[0].ssid, ssid, IEEE80211_MAX_SSID_LEN);
        local->int_scan_req->ssids[0].ssid_len = ssid_len;

        return __ieee80211_start_scan(sdata, sdata->local->int_scan_req);
}

void ieee80211_scan_cancel(struct ieee80211_local *local)
{
        /* ensure a new scan cannot be queued */
        lockdep_assert_wiphy(local->hw.wiphy);

        /*
         * We are canceling software scan, or deferred scan that was not
         * yet really started (see __ieee80211_start_scan ).
         *
         * Regarding hardware scan:
         * - we can not call  __ieee80211_scan_completed() as when
         *   SCAN_HW_SCANNING bit is set this function change
         *   local->hw_scan_req to operate on 5G band, what race with
         *   driver which can use local->hw_scan_req
         *
         * - we can not cancel scan_work since driver can schedule it
         *   by ieee80211_scan_completed(..., true) to finish scan
         *
         * Hence we only call the cancel_hw_scan() callback, but the low-level
         * driver is still responsible for calling ieee80211_scan_completed()
         * after the scan was completed/aborted.
         */

        if (!local->scan_req)
                return;

        /*
         * We have a scan running and the driver already reported completion,
         * but the worker hasn't run yet or is stuck on the mutex - mark it as
         * cancelled.
         */
        if (test_bit(SCAN_HW_SCANNING, &local->scanning) &&
            test_bit(SCAN_COMPLETED, &local->scanning)) {
                set_bit(SCAN_HW_CANCELLED, &local->scanning);
                return;
        }

        if (test_bit(SCAN_HW_SCANNING, &local->scanning)) {
                /*
                 * Make sure that __ieee80211_scan_completed doesn't trigger a
                 * scan on another band.
                 */
                set_bit(SCAN_HW_CANCELLED, &local->scanning);
                if (local->ops->cancel_hw_scan)
                        drv_cancel_hw_scan(local,
                                rcu_dereference_protected(local->scan_sdata,
                                                lockdep_is_held(&local->hw.wiphy->mtx)));
                return;
        }

        wiphy_delayed_work_cancel(local->hw.wiphy, &local->scan_work);
        /* and clean up */
        memset(&local->scan_info, 0, sizeof(local->scan_info));
        __ieee80211_scan_completed(&local->hw, true);
}

int __ieee80211_request_sched_scan_start(struct ieee80211_sub_if_data *sdata,
                                        struct cfg80211_sched_scan_request *req)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_scan_ies sched_scan_ies = {};
        struct cfg80211_chan_def chandef;
        int ret, i, iebufsz, num_bands = 0;
        u32 rate_masks[NUM_NL80211_BANDS] = {};
        u8 bands_used = 0;
        u8 *ie;
        u32 flags = 0;

        lockdep_assert_wiphy(local->hw.wiphy);

        iebufsz = local->scan_ies_len + req->ie_len;

        if (!local->ops->sched_scan_start)
                return -EOPNOTSUPP;

        for (i = 0; i < NUM_NL80211_BANDS; i++) {
                if (local->hw.wiphy->bands[i]) {
                        bands_used |= BIT(i);
                        rate_masks[i] = (u32) -1;
                        num_bands++;
                }
        }

        if (req->flags & NL80211_SCAN_FLAG_MIN_PREQ_CONTENT)
                flags |= IEEE80211_PROBE_FLAG_MIN_CONTENT;

        ie = kcalloc(iebufsz, num_bands, GFP_KERNEL);
        if (!ie) {
                ret = -ENOMEM;
                goto out;
        }

        ieee80211_prepare_scan_chandef(&chandef);

        ret = ieee80211_build_preq_ies(sdata, ie, num_bands * iebufsz,
                                       &sched_scan_ies, req->ie,
                                       req->ie_len, bands_used, rate_masks,
                                       &chandef, flags);
        if (ret < 0)
                goto error;

        ret = drv_sched_scan_start(local, sdata, req, &sched_scan_ies);
        if (ret == 0) {
                rcu_assign_pointer(local->sched_scan_sdata, sdata);
                rcu_assign_pointer(local->sched_scan_req, req);
        }

error:
        kfree(ie);
out:
        if (ret) {
                /* Clean in case of failure after HW restart or upon resume. */
                RCU_INIT_POINTER(local->sched_scan_sdata, NULL);
                RCU_INIT_POINTER(local->sched_scan_req, NULL);
        }

        return ret;
}

int ieee80211_request_sched_scan_start(struct ieee80211_sub_if_data *sdata,
                                       struct cfg80211_sched_scan_request *req)
{
        struct ieee80211_local *local = sdata->local;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (rcu_access_pointer(local->sched_scan_sdata))
                return -EBUSY;

        return __ieee80211_request_sched_scan_start(sdata, req);
}

int ieee80211_request_sched_scan_stop(struct ieee80211_local *local)
{
        struct ieee80211_sub_if_data *sched_scan_sdata;
        int ret = -ENOENT;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (!local->ops->sched_scan_stop)
                return -EOPNOTSUPP;

        /* We don't want to restart sched scan anymore. */
        RCU_INIT_POINTER(local->sched_scan_req, NULL);

        sched_scan_sdata = rcu_dereference_protected(local->sched_scan_sdata,
                                                lockdep_is_held(&local->hw.wiphy->mtx));
        if (sched_scan_sdata) {
                ret = drv_sched_scan_stop(local, sched_scan_sdata);
                if (!ret)
                        RCU_INIT_POINTER(local->sched_scan_sdata, NULL);
        }

        return ret;
}

void ieee80211_sched_scan_results(struct ieee80211_hw *hw)
{
        struct ieee80211_local *local = hw_to_local(hw);

        trace_api_sched_scan_results(local);

        cfg80211_sched_scan_results(hw->wiphy, 0);
}
EXPORT_SYMBOL(ieee80211_sched_scan_results);

void ieee80211_sched_scan_end(struct ieee80211_local *local)
{
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!rcu_access_pointer(local->sched_scan_sdata))
                return;

        RCU_INIT_POINTER(local->sched_scan_sdata, NULL);

        /* If sched scan was aborted by the driver. */
        RCU_INIT_POINTER(local->sched_scan_req, NULL);

        cfg80211_sched_scan_stopped_locked(local->hw.wiphy, 0);
}

void ieee80211_sched_scan_stopped_work(struct wiphy *wiphy,
                                       struct wiphy_work *work)
{
        struct ieee80211_local *local =
                container_of(work, struct ieee80211_local,
                             sched_scan_stopped_work);

        ieee80211_sched_scan_end(local);
}

void ieee80211_sched_scan_stopped(struct ieee80211_hw *hw)
{
        struct ieee80211_local *local = hw_to_local(hw);

        trace_api_sched_scan_stopped(local);

        /*
         * this shouldn't really happen, so for simplicity
         * simply ignore it, and let mac80211 reconfigure
         * the sched scan later on.
         */
        if (local->in_reconfig)
                return;

        wiphy_work_queue(hw->wiphy, &local->sched_scan_stopped_work);
}
EXPORT_SYMBOL(ieee80211_sched_scan_stopped);
































































   68 

































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
// SPDX-License-Identifier: GPL-2.0
#include <net/genetlink.h>
#include <net/netns/generic.h>
#include <uapi/linux/genetlink.h>
#include "ila.h"

static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
        [ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
        [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
        [ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
        [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
        [ILA_ATTR_IDENT_TYPE] = { .type = NLA_U8, },
};

static const struct genl_ops ila_nl_ops[] = {
        {
                .cmd = ILA_CMD_ADD,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit = ila_xlat_nl_cmd_add_mapping,
                .flags = GENL_ADMIN_PERM,
        },
        {
                .cmd = ILA_CMD_DEL,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit = ila_xlat_nl_cmd_del_mapping,
                .flags = GENL_ADMIN_PERM,
        },
        {
                .cmd = ILA_CMD_FLUSH,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit = ila_xlat_nl_cmd_flush,
                .flags = GENL_ADMIN_PERM,
        },
        {
                .cmd = ILA_CMD_GET,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit = ila_xlat_nl_cmd_get_mapping,
                .start = ila_xlat_nl_dump_start,
                .dumpit = ila_xlat_nl_dump,
                .done = ila_xlat_nl_dump_done,
        },
};

unsigned int ila_net_id;

struct genl_family ila_nl_family __ro_after_init = {
        .hdrsize        = 0,
        .name                = ILA_GENL_NAME,
        .version        = ILA_GENL_VERSION,
        .maxattr        = ILA_ATTR_MAX,
        .policy = ila_nl_policy,
        .netnsok        = true,
        .parallel_ops        = true,
        .module                = THIS_MODULE,
        .ops                = ila_nl_ops,
        .n_ops                = ARRAY_SIZE(ila_nl_ops),
        .resv_start_op        = ILA_CMD_FLUSH + 1,
};

static __net_init int ila_init_net(struct net *net)
{
        int err;

        err = ila_xlat_init_net(net);
        if (err)
                goto ila_xlat_init_fail;

        return 0;

ila_xlat_init_fail:
        return err;
}

static __net_exit void ila_pre_exit_net(struct net *net)
{
        ila_xlat_pre_exit_net(net);
}

static __net_exit void ila_exit_net(struct net *net)
{
        ila_xlat_exit_net(net);
}

static struct pernet_operations ila_net_ops = {
        .init = ila_init_net,
        .pre_exit = ila_pre_exit_net,
        .exit = ila_exit_net,
        .id   = &ila_net_id,
        .size = sizeof(struct ila_net),
};

static int __init ila_init(void)
{
        int ret;

        ret = register_pernet_device(&ila_net_ops);
        if (ret)
                goto register_device_fail;

        ret = genl_register_family(&ila_nl_family);
        if (ret)
                goto register_family_fail;

        ret = ila_lwt_init();
        if (ret)
                goto fail_lwt;

        return 0;

fail_lwt:
        genl_unregister_family(&ila_nl_family);
register_family_fail:
        unregister_pernet_device(&ila_net_ops);
register_device_fail:
        return ret;
}

static void __exit ila_fini(void)
{
        ila_lwt_fini();
        genl_unregister_family(&ila_nl_family);
        unregister_pernet_device(&ila_net_ops);
}

module_init(ila_init);
module_exit(ila_fini);
MODULE_AUTHOR("Tom Herbert <tom@herbertland.com>");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("IPv6: Identifier Locator Addressing (ILA)");






























































































    3 























    3 


    3 


    3 


    3 









    3 




    3 
    3 















    3 


    3 
    3 



    3 


    3 







    3 
    3 




















    4 































    1 



    1 
    1 


    1 






    1 
























































































    2 



    2 

    2 

    2 
    2 



















    3 

    3 



    3 


    3 
    3 

    3 


    3 







    3 

















    3 


    3 



    3 










    3 









    3 
    3 



    3 




































































































































































































































































































































































    3 



    3 
    3 








    3 





































































































































































































    3 













    3 
































































































































































































    3 




    3 


    3 







    3 







    3 

    3 


    3 








    3 







    3 




    3 



    3 

    3 










    3 




    3 



    3 

    3 



    3 




    3 


















    3 
    3 







    3 

    3 
    3 


    3 

    3 
    3 








    3 







    3 
    3 



















    3 



















    3 












    3 
























    3 

    3 
    3 




    3 









    3 
    3 






    3 
    3 






    3 

    3 
    3 






































































   37 



   37 





   38 



   38 




    7 








    3 





    3 

    3 


    3 
    3 



    3 



    3 
    3 

    3 


    3 


    3 

    3 

    3 


    3 

    3 


    3 




















































































































































































































































































































































































































































































































































































































































































































    3 










































    3 














































    4 





    3 








































































    4 
















    4 













    4 













    4 


    4 
    4 



    4 
    4 
    4 



    3 
    4 
    4 



    4 
    4 
    4 

    4 



    4 











    4 








    4 
















    4 
    4 












    4 


    4 



    4 

    4 
    4 


    4 
    4 

    4 
    4 


    4 

    4 











    4 


    4 
    4 









    4 


    3 
    4 



    4 
    4 



    4 

    4 
    4 


    3 

    4 
    4 


    4 
    4 



    4 









    4 






    4 

    4 
    4 



    3 
    4 

    4 








    3 
















    4 
    4 
    4 





    4 

    4 



    4 

    4 


    4 

























    3 




    3 

    4 











    1 
    4 

    4 
    4 
    4 

    4 
    1 
    3 
    1 
    4 


    3 


    4 



    4 
    4 




    4 








    4 








    4 





    3 


    4 
    4 



    4 







    4 





































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2002-2005, Instant802 Networks, Inc.
 * Copyright 2006-2007        Jiri Benc <jbenc@suse.cz>
 * Copyright 2013-2014  Intel Mobile Communications GmbH
 * Copyright (C) 2015 - 2017 Intel Deutschland GmbH
 * Copyright (C) 2018-2023 Intel Corporation
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/etherdevice.h>
#include <linux/netdevice.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/skbuff.h>
#include <linux/if_arp.h>
#include <linux/timer.h>
#include <linux/rtnetlink.h>

#include <net/codel.h>
#include <net/mac80211.h>
#include "ieee80211_i.h"
#include "driver-ops.h"
#include "rate.h"
#include "sta_info.h"
#include "debugfs_sta.h"
#include "mesh.h"
#include "wme.h"

/**
 * DOC: STA information lifetime rules
 *
 * STA info structures (&struct sta_info) are managed in a hash table
 * for faster lookup and a list for iteration. They are managed using
 * RCU, i.e. access to the list and hash table is protected by RCU.
 *
 * Upon allocating a STA info structure with sta_info_alloc(), the caller
 * owns that structure. It must then insert it into the hash table using
 * either sta_info_insert() or sta_info_insert_rcu(); only in the latter
 * case (which acquires an rcu read section but must not be called from
 * within one) will the pointer still be valid after the call. Note that
 * the caller may not do much with the STA info before inserting it; in
 * particular, it may not start any mesh peer link management or add
 * encryption keys.
 *
 * When the insertion fails (sta_info_insert()) returns non-zero), the
 * structure will have been freed by sta_info_insert()!
 *
 * Station entries are added by mac80211 when you establish a link with a
 * peer. This means different things for the different type of interfaces
 * we support. For a regular station this mean we add the AP sta when we
 * receive an association response from the AP. For IBSS this occurs when
 * get to know about a peer on the same IBSS. For WDS we add the sta for
 * the peer immediately upon device open. When using AP mode we add stations
 * for each respective station upon request from userspace through nl80211.
 *
 * In order to remove a STA info structure, various sta_info_destroy_*()
 * calls are available.
 *
 * There is no concept of ownership on a STA entry; each structure is
 * owned by the global hash table/list until it is removed. All users of
 * the structure need to be RCU protected so that the structure won't be
 * freed before they are done using it.
 */

struct sta_link_alloc {
        struct link_sta_info info;
        struct ieee80211_link_sta sta;
        struct rcu_head rcu_head;
};

static const struct rhashtable_params sta_rht_params = {
        .nelem_hint = 3, /* start small */
        .automatic_shrinking = true,
        .head_offset = offsetof(struct sta_info, hash_node),
        .key_offset = offsetof(struct sta_info, addr),
        .key_len = ETH_ALEN,
        .max_size = CONFIG_MAC80211_STA_HASH_MAX_SIZE,
};

static const struct rhashtable_params link_sta_rht_params = {
        .nelem_hint = 3, /* start small */
        .automatic_shrinking = true,
        .head_offset = offsetof(struct link_sta_info, link_hash_node),
        .key_offset = offsetof(struct link_sta_info, addr),
        .key_len = ETH_ALEN,
        .max_size = CONFIG_MAC80211_STA_HASH_MAX_SIZE,
};

static int sta_info_hash_del(struct ieee80211_local *local,
                             struct sta_info *sta)
{
        return rhltable_remove(&local->sta_hash, &sta->hash_node,
                               sta_rht_params);
}

static int link_sta_info_hash_add(struct ieee80211_local *local,
                                  struct link_sta_info *link_sta)
{
        lockdep_assert_wiphy(local->hw.wiphy);

        return rhltable_insert(&local->link_sta_hash,
                               &link_sta->link_hash_node, link_sta_rht_params);
}

static int link_sta_info_hash_del(struct ieee80211_local *local,
                                  struct link_sta_info *link_sta)
{
        lockdep_assert_wiphy(local->hw.wiphy);

        return rhltable_remove(&local->link_sta_hash,
                               &link_sta->link_hash_node, link_sta_rht_params);
}

void ieee80211_purge_sta_txqs(struct sta_info *sta)
{
        struct ieee80211_local *local = sta->sdata->local;
        int i;

        for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
                struct txq_info *txqi;

                if (!sta->sta.txq[i])
                        continue;

                txqi = to_txq_info(sta->sta.txq[i]);

                ieee80211_txq_purge(local, txqi);
        }
}

static void __cleanup_single_sta(struct sta_info *sta)
{
        int ac, i;
        struct tid_ampdu_tx *tid_tx;
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ps_data *ps;

        if (test_sta_flag(sta, WLAN_STA_PS_STA) ||
            test_sta_flag(sta, WLAN_STA_PS_DRIVER) ||
            test_sta_flag(sta, WLAN_STA_PS_DELIVER)) {
                if (sta->sdata->vif.type == NL80211_IFTYPE_AP ||
                    sta->sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
                        ps = &sdata->bss->ps;
                else if (ieee80211_vif_is_mesh(&sdata->vif))
                        ps = &sdata->u.mesh.ps;
                else
                        return;

                clear_sta_flag(sta, WLAN_STA_PS_STA);
                clear_sta_flag(sta, WLAN_STA_PS_DRIVER);
                clear_sta_flag(sta, WLAN_STA_PS_DELIVER);

                atomic_dec(&ps->num_sta_ps);
        }

        ieee80211_purge_sta_txqs(sta);

        for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
                local->total_ps_buffered -= skb_queue_len(&sta->ps_tx_buf[ac]);
                ieee80211_purge_tx_queue(&local->hw, &sta->ps_tx_buf[ac]);
                ieee80211_purge_tx_queue(&local->hw, &sta->tx_filtered[ac]);
        }

        if (ieee80211_vif_is_mesh(&sdata->vif))
                mesh_sta_cleanup(sta);

        cancel_work_sync(&sta->drv_deliver_wk);

        /*
         * Destroy aggregation state here. It would be nice to wait for the
         * driver to finish aggregation stop and then clean up, but for now
         * drivers have to handle aggregation stop being requested, followed
         * directly by station destruction.
         */
        for (i = 0; i < IEEE80211_NUM_TIDS; i++) {
                kfree(sta->ampdu_mlme.tid_start_tx[i]);
                tid_tx = rcu_dereference_raw(sta->ampdu_mlme.tid_tx[i]);
                if (!tid_tx)
                        continue;
                ieee80211_purge_tx_queue(&local->hw, &tid_tx->pending);
                kfree(tid_tx);
        }
}

static void cleanup_single_sta(struct sta_info *sta)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;

        __cleanup_single_sta(sta);
        sta_info_free(local, sta);
}

struct rhlist_head *sta_info_hash_lookup(struct ieee80211_local *local,
                                         const u8 *addr)
{
        return rhltable_lookup(&local->sta_hash, addr, sta_rht_params);
}

/* protected by RCU */
struct sta_info *sta_info_get(struct ieee80211_sub_if_data *sdata,
                              const u8 *addr)
{
        struct ieee80211_local *local = sdata->local;
        struct rhlist_head *tmp;
        struct sta_info *sta;

        rcu_read_lock();
        for_each_sta_info(local, addr, sta, tmp) {
                if (sta->sdata == sdata) {
                        rcu_read_unlock();
                        /* this is safe as the caller must already hold
                         * another rcu read section or the mutex
                         */
                        return sta;
                }
        }
        rcu_read_unlock();
        return NULL;
}

/*
 * Get sta info either from the specified interface
 * or from one of its vlans
 */
struct sta_info *sta_info_get_bss(struct ieee80211_sub_if_data *sdata,
                                  const u8 *addr)
{
        struct ieee80211_local *local = sdata->local;
        struct rhlist_head *tmp;
        struct sta_info *sta;

        rcu_read_lock();
        for_each_sta_info(local, addr, sta, tmp) {
                if (sta->sdata == sdata ||
                    (sta->sdata->bss && sta->sdata->bss == sdata->bss)) {
                        rcu_read_unlock();
                        /* this is safe as the caller must already hold
                         * another rcu read section or the mutex
                         */
                        return sta;
                }
        }
        rcu_read_unlock();
        return NULL;
}

struct rhlist_head *link_sta_info_hash_lookup(struct ieee80211_local *local,
                                              const u8 *addr)
{
        return rhltable_lookup(&local->link_sta_hash, addr,
                               link_sta_rht_params);
}

struct link_sta_info *
link_sta_info_get_bss(struct ieee80211_sub_if_data *sdata, const u8 *addr)
{
        struct ieee80211_local *local = sdata->local;
        struct rhlist_head *tmp;
        struct link_sta_info *link_sta;

        rcu_read_lock();
        for_each_link_sta_info(local, addr, link_sta, tmp) {
                struct sta_info *sta = link_sta->sta;

                if (sta->sdata == sdata ||
                    (sta->sdata->bss && sta->sdata->bss == sdata->bss)) {
                        rcu_read_unlock();
                        /* this is safe as the caller must already hold
                         * another rcu read section or the mutex
                         */
                        return link_sta;
                }
        }
        rcu_read_unlock();
        return NULL;
}

struct ieee80211_sta *
ieee80211_find_sta_by_link_addrs(struct ieee80211_hw *hw,
                                 const u8 *addr,
                                 const u8 *localaddr,
                                 unsigned int *link_id)
{
        struct ieee80211_local *local = hw_to_local(hw);
        struct link_sta_info *link_sta;
        struct rhlist_head *tmp;

        for_each_link_sta_info(local, addr, link_sta, tmp) {
                struct sta_info *sta = link_sta->sta;
                struct ieee80211_link_data *link;
                u8 _link_id = link_sta->link_id;

                if (!localaddr) {
                        if (link_id)
                                *link_id = _link_id;
                        return &sta->sta;
                }

                link = rcu_dereference(sta->sdata->link[_link_id]);
                if (!link)
                        continue;

                if (memcmp(link->conf->addr, localaddr, ETH_ALEN))
                        continue;

                if (link_id)
                        *link_id = _link_id;
                return &sta->sta;
        }

        return NULL;
}
EXPORT_SYMBOL_GPL(ieee80211_find_sta_by_link_addrs);

struct sta_info *sta_info_get_by_addrs(struct ieee80211_local *local,
                                       const u8 *sta_addr, const u8 *vif_addr)
{
        struct rhlist_head *tmp;
        struct sta_info *sta;

        for_each_sta_info(local, sta_addr, sta, tmp) {
                if (ether_addr_equal(vif_addr, sta->sdata->vif.addr))
                        return sta;
        }

        return NULL;
}

struct sta_info *sta_info_get_by_idx(struct ieee80211_sub_if_data *sdata,
                                     int idx)
{
        struct ieee80211_local *local = sdata->local;
        struct sta_info *sta;
        int i = 0;

        list_for_each_entry_rcu(sta, &local->sta_list, list,
                                lockdep_is_held(&local->hw.wiphy->mtx)) {
                if (sdata != sta->sdata)
                        continue;
                if (i < idx) {
                        ++i;
                        continue;
                }
                return sta;
        }

        return NULL;
}

static void sta_info_free_link(struct link_sta_info *link_sta)
{
        free_percpu(link_sta->pcpu_rx_stats);
}

static void sta_remove_link(struct sta_info *sta, unsigned int link_id,
                            bool unhash)
{
        struct sta_link_alloc *alloc = NULL;
        struct link_sta_info *link_sta;

        lockdep_assert_wiphy(sta->local->hw.wiphy);

        link_sta = rcu_access_pointer(sta->link[link_id]);
        if (WARN_ON(!link_sta))
                return;

        if (unhash)
                link_sta_info_hash_del(sta->local, link_sta);

        if (test_sta_flag(sta, WLAN_STA_INSERTED))
                ieee80211_link_sta_debugfs_remove(link_sta);

        if (link_sta != &sta->deflink)
                alloc = container_of(link_sta, typeof(*alloc), info);

        sta->sta.valid_links &= ~BIT(link_id);
        RCU_INIT_POINTER(sta->link[link_id], NULL);
        RCU_INIT_POINTER(sta->sta.link[link_id], NULL);
        if (alloc) {
                sta_info_free_link(&alloc->info);
                kfree_rcu(alloc, rcu_head);
        }

        ieee80211_sta_recalc_aggregates(&sta->sta);
}

/**
 * sta_info_free - free STA
 *
 * @local: pointer to the global information
 * @sta: STA info to free
 *
 * This function must undo everything done by sta_info_alloc()
 * that may happen before sta_info_insert(). It may only be
 * called when sta_info_insert() has not been attempted (and
 * if that fails, the station is freed anyway.)
 */
void sta_info_free(struct ieee80211_local *local, struct sta_info *sta)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(sta->link); i++) {
                struct link_sta_info *link_sta;

                link_sta = rcu_access_pointer(sta->link[i]);
                if (!link_sta)
                        continue;

                sta_remove_link(sta, i, false);
        }

        /*
         * If we had used sta_info_pre_move_state() then we might not
         * have gone through the state transitions down again, so do
         * it here now (and warn if it's inserted).
         *
         * This will clear state such as fast TX/RX that may have been
         * allocated during state transitions.
         */
        while (sta->sta_state > IEEE80211_STA_NONE) {
                int ret;

                WARN_ON_ONCE(test_sta_flag(sta, WLAN_STA_INSERTED));

                ret = sta_info_move_state(sta, sta->sta_state - 1);
                if (WARN_ONCE(ret, "sta_info_move_state() returned %d\n", ret))
                        break;
        }

        if (sta->rate_ctrl)
                rate_control_free_sta(sta);

        sta_dbg(sta->sdata, "Destroyed STA %pM\n", sta->sta.addr);

        kfree(to_txq_info(sta->sta.txq[0]));
        kfree(rcu_dereference_raw(sta->sta.rates));
#ifdef CONFIG_MAC80211_MESH
        kfree(sta->mesh);
#endif

        sta_info_free_link(&sta->deflink);
        kfree(sta);
}

static int sta_info_hash_add(struct ieee80211_local *local,
                             struct sta_info *sta)
{
        return rhltable_insert(&local->sta_hash, &sta->hash_node,
                               sta_rht_params);
}

static void sta_deliver_ps_frames(struct work_struct *wk)
{
        struct sta_info *sta;

        sta = container_of(wk, struct sta_info, drv_deliver_wk);

        if (sta->dead)
                return;

        local_bh_disable();
        if (!test_sta_flag(sta, WLAN_STA_PS_STA))
                ieee80211_sta_ps_deliver_wakeup(sta);
        else if (test_and_clear_sta_flag(sta, WLAN_STA_PSPOLL))
                ieee80211_sta_ps_deliver_poll_response(sta);
        else if (test_and_clear_sta_flag(sta, WLAN_STA_UAPSD))
                ieee80211_sta_ps_deliver_uapsd(sta);
        local_bh_enable();
}

static int sta_prepare_rate_control(struct ieee80211_local *local,
                                    struct sta_info *sta, gfp_t gfp)
{
        if (ieee80211_hw_check(&local->hw, HAS_RATE_CONTROL))
                return 0;

        sta->rate_ctrl = local->rate_ctrl;
        sta->rate_ctrl_priv = rate_control_alloc_sta(sta->rate_ctrl,
                                                     sta, gfp);
        if (!sta->rate_ctrl_priv)
                return -ENOMEM;

        return 0;
}

static int sta_info_alloc_link(struct ieee80211_local *local,
                               struct link_sta_info *link_info,
                               gfp_t gfp)
{
        struct ieee80211_hw *hw = &local->hw;
        int i;

        if (ieee80211_hw_check(hw, USES_RSS)) {
                link_info->pcpu_rx_stats =
                        alloc_percpu_gfp(struct ieee80211_sta_rx_stats, gfp);
                if (!link_info->pcpu_rx_stats)
                        return -ENOMEM;
        }

        link_info->rx_stats.last_rx = jiffies;
        u64_stats_init(&link_info->rx_stats.syncp);

        ewma_signal_init(&link_info->rx_stats_avg.signal);
        ewma_avg_signal_init(&link_info->status_stats.avg_ack_signal);
        for (i = 0; i < ARRAY_SIZE(link_info->rx_stats_avg.chain_signal); i++)
                ewma_signal_init(&link_info->rx_stats_avg.chain_signal[i]);

        return 0;
}

static void sta_info_add_link(struct sta_info *sta,
                              unsigned int link_id,
                              struct link_sta_info *link_info,
                              struct ieee80211_link_sta *link_sta)
{
        link_info->sta = sta;
        link_info->link_id = link_id;
        link_info->pub = link_sta;
        link_info->pub->sta = &sta->sta;
        link_sta->link_id = link_id;
        rcu_assign_pointer(sta->link[link_id], link_info);
        rcu_assign_pointer(sta->sta.link[link_id], link_sta);

        link_sta->smps_mode = IEEE80211_SMPS_OFF;
        link_sta->agg.max_rc_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_BA;
}

static struct sta_info *
__sta_info_alloc(struct ieee80211_sub_if_data *sdata,
                 const u8 *addr, int link_id, const u8 *link_addr,
                 gfp_t gfp)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_hw *hw = &local->hw;
        struct sta_info *sta;
        void *txq_data;
        int size;
        int i;

        sta = kzalloc(sizeof(*sta) + hw->sta_data_size, gfp);
        if (!sta)
                return NULL;

        sta->local = local;
        sta->sdata = sdata;

        if (sta_info_alloc_link(local, &sta->deflink, gfp))
                goto free;

        if (link_id >= 0) {
                sta_info_add_link(sta, link_id, &sta->deflink,
                                  &sta->sta.deflink);
                sta->sta.valid_links = BIT(link_id);
        } else {
                sta_info_add_link(sta, 0, &sta->deflink, &sta->sta.deflink);
        }

        sta->sta.cur = &sta->sta.deflink.agg;

        spin_lock_init(&sta->lock);
        spin_lock_init(&sta->ps_lock);
        INIT_WORK(&sta->drv_deliver_wk, sta_deliver_ps_frames);
        wiphy_work_init(&sta->ampdu_mlme.work, ieee80211_ba_session_work);
#ifdef CONFIG_MAC80211_MESH
        if (ieee80211_vif_is_mesh(&sdata->vif)) {
                sta->mesh = kzalloc(sizeof(*sta->mesh), gfp);
                if (!sta->mesh)
                        goto free;
                sta->mesh->plink_sta = sta;
                spin_lock_init(&sta->mesh->plink_lock);
                if (!sdata->u.mesh.user_mpm)
                        timer_setup(&sta->mesh->plink_timer, mesh_plink_timer,
                                    0);
                sta->mesh->nonpeer_pm = NL80211_MESH_POWER_ACTIVE;
        }
#endif

        memcpy(sta->addr, addr, ETH_ALEN);
        memcpy(sta->sta.addr, addr, ETH_ALEN);
        memcpy(sta->deflink.addr, link_addr, ETH_ALEN);
        memcpy(sta->sta.deflink.addr, link_addr, ETH_ALEN);
        sta->sta.max_rx_aggregation_subframes =
                local->hw.max_rx_aggregation_subframes;

        /* TODO link specific alloc and assignments for MLO Link STA */

        /* Extended Key ID needs to install keys for keyid 0 and 1 Rx-only.
         * The Tx path starts to use a key as soon as the key slot ptk_idx
         * references to is not NULL. To not use the initial Rx-only key
         * prematurely for Tx initialize ptk_idx to an impossible PTK keyid
         * which always will refer to a NULL key.
         */
        BUILD_BUG_ON(ARRAY_SIZE(sta->ptk) <= INVALID_PTK_KEYIDX);
        sta->ptk_idx = INVALID_PTK_KEYIDX;


        ieee80211_init_frag_cache(&sta->frags);

        sta->sta_state = IEEE80211_STA_NONE;

        if (sdata->vif.type == NL80211_IFTYPE_MESH_POINT)
                sta->amsdu_mesh_control = -1;

        /* Mark TID as unreserved */
        sta->reserved_tid = IEEE80211_TID_UNRESERVED;

        sta->last_connected = ktime_get_seconds();

        size = sizeof(struct txq_info) +
               ALIGN(hw->txq_data_size, sizeof(void *));

        txq_data = kcalloc(ARRAY_SIZE(sta->sta.txq), size, gfp);
        if (!txq_data)
                goto free;

        for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
                struct txq_info *txq = txq_data + i * size;

                /* might not do anything for the (bufferable) MMPDU TXQ */
                ieee80211_txq_init(sdata, sta, txq, i);
        }

        if (sta_prepare_rate_control(local, sta, gfp))
                goto free_txq;

        sta->airtime_weight = IEEE80211_DEFAULT_AIRTIME_WEIGHT;

        for (i = 0; i < IEEE80211_NUM_ACS; i++) {
                skb_queue_head_init(&sta->ps_tx_buf[i]);
                skb_queue_head_init(&sta->tx_filtered[i]);
                sta->airtime[i].deficit = sta->airtime_weight;
                atomic_set(&sta->airtime[i].aql_tx_pending, 0);
                sta->airtime[i].aql_limit_low = local->aql_txq_limit_low[i];
                sta->airtime[i].aql_limit_high = local->aql_txq_limit_high[i];
        }

        for (i = 0; i < IEEE80211_NUM_TIDS; i++)
                sta->last_seq_ctrl[i] = cpu_to_le16(USHRT_MAX);

        for (i = 0; i < NUM_NL80211_BANDS; i++) {
                u32 mandatory = 0;
                int r;

                if (!hw->wiphy->bands[i])
                        continue;

                switch (i) {
                case NL80211_BAND_2GHZ:
                case NL80211_BAND_LC:
                        /*
                         * We use both here, even if we cannot really know for
                         * sure the station will support both, but the only use
                         * for this is when we don't know anything yet and send
                         * management frames, and then we'll pick the lowest
                         * possible rate anyway.
                         * If we don't include _G here, we cannot find a rate
                         * in P2P, and thus trigger the WARN_ONCE() in rate.c
                         */
                        mandatory = IEEE80211_RATE_MANDATORY_B |
                                    IEEE80211_RATE_MANDATORY_G;
                        break;
                case NL80211_BAND_5GHZ:
                        mandatory = IEEE80211_RATE_MANDATORY_A;
                        break;
                case NL80211_BAND_60GHZ:
                        WARN_ON(1);
                        mandatory = 0;
                        break;
                }

                for (r = 0; r < hw->wiphy->bands[i]->n_bitrates; r++) {
                        struct ieee80211_rate *rate;

                        rate = &hw->wiphy->bands[i]->bitrates[r];

                        if (!(rate->flags & mandatory))
                                continue;
                        sta->sta.deflink.supp_rates[i] |= BIT(r);
                }
        }

        sta->cparams.ce_threshold = CODEL_DISABLED_THRESHOLD;
        sta->cparams.target = MS2TIME(20);
        sta->cparams.interval = MS2TIME(100);
        sta->cparams.ecn = true;
        sta->cparams.ce_threshold_selector = 0;
        sta->cparams.ce_threshold_mask = 0;

        sta_dbg(sdata, "Allocated STA %pM\n", sta->sta.addr);

        return sta;

free_txq:
        kfree(to_txq_info(sta->sta.txq[0]));
free:
        sta_info_free_link(&sta->deflink);
#ifdef CONFIG_MAC80211_MESH
        kfree(sta->mesh);
#endif
        kfree(sta);
        return NULL;
}

struct sta_info *sta_info_alloc(struct ieee80211_sub_if_data *sdata,
                                const u8 *addr, gfp_t gfp)
{
        return __sta_info_alloc(sdata, addr, -1, addr, gfp);
}

struct sta_info *sta_info_alloc_with_link(struct ieee80211_sub_if_data *sdata,
                                          const u8 *mld_addr,
                                          unsigned int link_id,
                                          const u8 *link_addr,
                                          gfp_t gfp)
{
        return __sta_info_alloc(sdata, mld_addr, link_id, link_addr, gfp);
}

static int sta_info_insert_check(struct sta_info *sta)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        /*
         * Can't be a WARN_ON because it can be triggered through a race:
         * something inserts a STA (on one CPU) without holding the RTNL
         * and another CPU turns off the net device.
         */
        if (unlikely(!ieee80211_sdata_running(sdata)))
                return -ENETDOWN;

        if (WARN_ON(ether_addr_equal(sta->sta.addr, sdata->vif.addr) ||
                    !is_valid_ether_addr(sta->sta.addr)))
                return -EINVAL;

        /* The RCU read lock is required by rhashtable due to
         * asynchronous resize/rehash.  We also require the mutex
         * for correctness.
         */
        rcu_read_lock();
        if (ieee80211_hw_check(&sdata->local->hw, NEEDS_UNIQUE_STA_ADDR) &&
            ieee80211_find_sta_by_ifaddr(&sdata->local->hw, sta->addr, NULL)) {
                rcu_read_unlock();
                return -ENOTUNIQ;
        }
        rcu_read_unlock();

        return 0;
}

static int sta_info_insert_drv_state(struct ieee80211_local *local,
                                     struct ieee80211_sub_if_data *sdata,
                                     struct sta_info *sta)
{
        enum ieee80211_sta_state state;
        int err = 0;

        for (state = IEEE80211_STA_NOTEXIST; state < sta->sta_state; state++) {
                err = drv_sta_state(local, sdata, sta, state, state + 1);
                if (err)
                        break;
        }

        if (!err) {
                /*
                 * Drivers using legacy sta_add/sta_remove callbacks only
                 * get uploaded set to true after sta_add is called.
                 */
                if (!local->ops->sta_add)
                        sta->uploaded = true;
                return 0;
        }

        if (sdata->vif.type == NL80211_IFTYPE_ADHOC) {
                sdata_info(sdata,
                           "failed to move IBSS STA %pM to state %d (%d) - keeping it anyway\n",
                           sta->sta.addr, state + 1, err);
                err = 0;
        }

        /* unwind on error */
        for (; state > IEEE80211_STA_NOTEXIST; state--)
                WARN_ON(drv_sta_state(local, sdata, sta, state, state - 1));

        return err;
}

static void
ieee80211_recalc_p2p_go_ps_allowed(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        bool allow_p2p_go_ps = sdata->vif.p2p;
        struct sta_info *sta;

        rcu_read_lock();
        list_for_each_entry_rcu(sta, &local->sta_list, list) {
                if (sdata != sta->sdata ||
                    !test_sta_flag(sta, WLAN_STA_ASSOC))
                        continue;
                if (!sta->sta.support_p2p_ps) {
                        allow_p2p_go_ps = false;
                        break;
                }
        }
        rcu_read_unlock();

        if (allow_p2p_go_ps != sdata->vif.bss_conf.allow_p2p_go_ps) {
                sdata->vif.bss_conf.allow_p2p_go_ps = allow_p2p_go_ps;
                ieee80211_link_info_change_notify(sdata, &sdata->deflink,
                                                  BSS_CHANGED_P2P_PS);
        }
}

static int sta_info_insert_finish(struct sta_info *sta) __acquires(RCU)
{
        struct ieee80211_local *local = sta->local;
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct station_info *sinfo = NULL;
        int err = 0;

        lockdep_assert_wiphy(local->hw.wiphy);

        /* check if STA exists already */
        if (sta_info_get_bss(sdata, sta->sta.addr)) {
                err = -EEXIST;
                goto out_cleanup;
        }

        sinfo = kzalloc(sizeof(struct station_info), GFP_KERNEL);
        if (!sinfo) {
                err = -ENOMEM;
                goto out_cleanup;
        }

        local->num_sta++;
        local->sta_generation++;
        smp_mb();

        /* simplify things and don't accept BA sessions yet */
        set_sta_flag(sta, WLAN_STA_BLOCK_BA);

        /* make the station visible */
        err = sta_info_hash_add(local, sta);
        if (err)
                goto out_drop_sta;

        if (sta->sta.valid_links) {
                err = link_sta_info_hash_add(local, &sta->deflink);
                if (err) {
                        sta_info_hash_del(local, sta);
                        goto out_drop_sta;
                }
        }

        list_add_tail_rcu(&sta->list, &local->sta_list);

        /* update channel context before notifying the driver about state
         * change, this enables driver using the updated channel context right away.
         */
        if (sta->sta_state >= IEEE80211_STA_ASSOC) {
                ieee80211_recalc_min_chandef(sta->sdata, -1);
                if (!sta->sta.support_p2p_ps)
                        ieee80211_recalc_p2p_go_ps_allowed(sta->sdata);
        }

        /* notify driver */
        err = sta_info_insert_drv_state(local, sdata, sta);
        if (err)
                goto out_remove;

        set_sta_flag(sta, WLAN_STA_INSERTED);

        /* accept BA sessions now */
        clear_sta_flag(sta, WLAN_STA_BLOCK_BA);

        ieee80211_sta_debugfs_add(sta);
        rate_control_add_sta_debugfs(sta);
        if (sta->sta.valid_links) {
                int i;

                for (i = 0; i < ARRAY_SIZE(sta->link); i++) {
                        struct link_sta_info *link_sta;

                        link_sta = rcu_dereference_protected(sta->link[i],
                                                             lockdep_is_held(&local->hw.wiphy->mtx));

                        if (!link_sta)
                                continue;

                        ieee80211_link_sta_debugfs_add(link_sta);
                        if (sdata->vif.active_links & BIT(i))
                                ieee80211_link_sta_debugfs_drv_add(link_sta);
                }
        } else {
                ieee80211_link_sta_debugfs_add(&sta->deflink);
                ieee80211_link_sta_debugfs_drv_add(&sta->deflink);
        }

        sinfo->generation = local->sta_generation;
        cfg80211_new_sta(sdata->dev, sta->sta.addr, sinfo, GFP_KERNEL);
        kfree(sinfo);

        sta_dbg(sdata, "Inserted STA %pM\n", sta->sta.addr);

        /* move reference to rcu-protected */
        rcu_read_lock();

        if (ieee80211_vif_is_mesh(&sdata->vif))
                mesh_accept_plinks_update(sdata);

        ieee80211_check_fast_xmit(sta);

        return 0;
 out_remove:
        if (sta->sta.valid_links)
                link_sta_info_hash_del(local, &sta->deflink);
        sta_info_hash_del(local, sta);
        list_del_rcu(&sta->list);
 out_drop_sta:
        local->num_sta--;
        synchronize_net();
 out_cleanup:
        cleanup_single_sta(sta);
        kfree(sinfo);
        rcu_read_lock();
        return err;
}

int sta_info_insert_rcu(struct sta_info *sta) __acquires(RCU)
{
        struct ieee80211_local *local = sta->local;
        int err;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        err = sta_info_insert_check(sta);
        if (err) {
                sta_info_free(local, sta);
                rcu_read_lock();
                return err;
        }

        return sta_info_insert_finish(sta);
}

int sta_info_insert(struct sta_info *sta)
{
        int err = sta_info_insert_rcu(sta);

        rcu_read_unlock();

        return err;
}

static inline void __bss_tim_set(u8 *tim, u16 id)
{
        /*
         * This format has been mandated by the IEEE specifications,
         * so this line may not be changed to use the __set_bit() format.
         */
        tim[id / 8] |= (1 << (id % 8));
}

static inline void __bss_tim_clear(u8 *tim, u16 id)
{
        /*
         * This format has been mandated by the IEEE specifications,
         * so this line may not be changed to use the __clear_bit() format.
         */
        tim[id / 8] &= ~(1 << (id % 8));
}

static inline bool __bss_tim_get(u8 *tim, u16 id)
{
        /*
         * This format has been mandated by the IEEE specifications,
         * so this line may not be changed to use the test_bit() format.
         */
        return tim[id / 8] & (1 << (id % 8));
}

static unsigned long ieee80211_tids_for_ac(int ac)
{
        /* If we ever support TIDs > 7, this obviously needs to be adjusted */
        switch (ac) {
        case IEEE80211_AC_VO:
                return BIT(6) | BIT(7);
        case IEEE80211_AC_VI:
                return BIT(4) | BIT(5);
        case IEEE80211_AC_BE:
                return BIT(0) | BIT(3);
        case IEEE80211_AC_BK:
                return BIT(1) | BIT(2);
        default:
                WARN_ON(1);
                return 0;
        }
}

static void __sta_info_recalc_tim(struct sta_info *sta, bool ignore_pending)
{
        struct ieee80211_local *local = sta->local;
        struct ps_data *ps;
        bool indicate_tim = false;
        u8 ignore_for_tim = sta->sta.uapsd_queues;
        int ac;
        u16 id = sta->sta.aid;

        if (sta->sdata->vif.type == NL80211_IFTYPE_AP ||
            sta->sdata->vif.type == NL80211_IFTYPE_AP_VLAN) {
                if (WARN_ON_ONCE(!sta->sdata->bss))
                        return;

                ps = &sta->sdata->bss->ps;
#ifdef CONFIG_MAC80211_MESH
        } else if (ieee80211_vif_is_mesh(&sta->sdata->vif)) {
                ps = &sta->sdata->u.mesh.ps;
#endif
        } else {
                return;
        }

        /* No need to do anything if the driver does all */
        if (ieee80211_hw_check(&local->hw, AP_LINK_PS) && !local->ops->set_tim)
                return;

        if (sta->dead)
                goto done;

        /*
         * If all ACs are delivery-enabled then we should build
         * the TIM bit for all ACs anyway; if only some are then
         * we ignore those and build the TIM bit using only the
         * non-enabled ones.
         */
        if (ignore_for_tim == BIT(IEEE80211_NUM_ACS) - 1)
                ignore_for_tim = 0;

        if (ignore_pending)
                ignore_for_tim = BIT(IEEE80211_NUM_ACS) - 1;

        for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
                unsigned long tids;

                if (ignore_for_tim & ieee80211_ac_to_qos_mask[ac])
                        continue;

                indicate_tim |= !skb_queue_empty(&sta->tx_filtered[ac]) ||
                                !skb_queue_empty(&sta->ps_tx_buf[ac]);
                if (indicate_tim)
                        break;

                tids = ieee80211_tids_for_ac(ac);

                indicate_tim |=
                        sta->driver_buffered_tids & tids;
                indicate_tim |=
                        sta->txq_buffered_tids & tids;
        }

 done:
        spin_lock_bh(&local->tim_lock);

        if (indicate_tim == __bss_tim_get(ps->tim, id))
                goto out_unlock;

        if (indicate_tim)
                __bss_tim_set(ps->tim, id);
        else
                __bss_tim_clear(ps->tim, id);

        if (local->ops->set_tim && !WARN_ON(sta->dead)) {
                local->tim_in_locked_section = true;
                drv_set_tim(local, &sta->sta, indicate_tim);
                local->tim_in_locked_section = false;
        }

out_unlock:
        spin_unlock_bh(&local->tim_lock);
}

void sta_info_recalc_tim(struct sta_info *sta)
{
        __sta_info_recalc_tim(sta, false);
}

static bool sta_info_buffer_expired(struct sta_info *sta, struct sk_buff *skb)
{
        struct ieee80211_tx_info *info;
        int timeout;

        if (!skb)
                return false;

        info = IEEE80211_SKB_CB(skb);

        /* Timeout: (2 * listen_interval * beacon_int * 1024 / 1000000) sec */
        timeout = (sta->listen_interval *
                   sta->sdata->vif.bss_conf.beacon_int *
                   32 / 15625) * HZ;
        if (timeout < STA_TX_BUFFER_EXPIRE)
                timeout = STA_TX_BUFFER_EXPIRE;
        return time_after(jiffies, info->control.jiffies + timeout);
}


static bool sta_info_cleanup_expire_buffered_ac(struct ieee80211_local *local,
                                                struct sta_info *sta, int ac)
{
        unsigned long flags;
        struct sk_buff *skb;

        /*
         * First check for frames that should expire on the filtered
         * queue. Frames here were rejected by the driver and are on
         * a separate queue to avoid reordering with normal PS-buffered
         * frames. They also aren't accounted for right now in the
         * total_ps_buffered counter.
         */
        for (;;) {
                spin_lock_irqsave(&sta->tx_filtered[ac].lock, flags);
                skb = skb_peek(&sta->tx_filtered[ac]);
                if (sta_info_buffer_expired(sta, skb))
                        skb = __skb_dequeue(&sta->tx_filtered[ac]);
                else
                        skb = NULL;
                spin_unlock_irqrestore(&sta->tx_filtered[ac].lock, flags);

                /*
                 * Frames are queued in order, so if this one
                 * hasn't expired yet we can stop testing. If
                 * we actually reached the end of the queue we
                 * also need to stop, of course.
                 */
                if (!skb)
                        break;
                ieee80211_free_txskb(&local->hw, skb);
        }

        /*
         * Now also check the normal PS-buffered queue, this will
         * only find something if the filtered queue was emptied
         * since the filtered frames are all before the normal PS
         * buffered frames.
         */
        for (;;) {
                spin_lock_irqsave(&sta->ps_tx_buf[ac].lock, flags);
                skb = skb_peek(&sta->ps_tx_buf[ac]);
                if (sta_info_buffer_expired(sta, skb))
                        skb = __skb_dequeue(&sta->ps_tx_buf[ac]);
                else
                        skb = NULL;
                spin_unlock_irqrestore(&sta->ps_tx_buf[ac].lock, flags);

                /*
                 * frames are queued in order, so if this one
                 * hasn't expired yet (or we reached the end of
                 * the queue) we can stop testing
                 */
                if (!skb)
                        break;

                local->total_ps_buffered--;
                ps_dbg(sta->sdata, "Buffered frame expired (STA %pM)\n",
                       sta->sta.addr);
                ieee80211_free_txskb(&local->hw, skb);
        }

        /*
         * Finally, recalculate the TIM bit for this station -- it might
         * now be clear because the station was too slow to retrieve its
         * frames.
         */
        sta_info_recalc_tim(sta);

        /*
         * Return whether there are any frames still buffered, this is
         * used to check whether the cleanup timer still needs to run,
         * if there are no frames we don't need to rearm the timer.
         */
        return !(skb_queue_empty(&sta->ps_tx_buf[ac]) &&
                 skb_queue_empty(&sta->tx_filtered[ac]));
}

static bool sta_info_cleanup_expire_buffered(struct ieee80211_local *local,
                                             struct sta_info *sta)
{
        bool have_buffered = false;
        int ac;

        /* This is only necessary for stations on BSS/MBSS interfaces */
        if (!sta->sdata->bss &&
            !ieee80211_vif_is_mesh(&sta->sdata->vif))
                return false;

        for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
                have_buffered |=
                        sta_info_cleanup_expire_buffered_ac(local, sta, ac);

        return have_buffered;
}

static int __must_check __sta_info_destroy_part1(struct sta_info *sta)
{
        struct ieee80211_local *local;
        struct ieee80211_sub_if_data *sdata;
        int ret, i;

        might_sleep();

        if (!sta)
                return -ENOENT;

        local = sta->local;
        sdata = sta->sdata;

        lockdep_assert_wiphy(local->hw.wiphy);

        /*
         * Before removing the station from the driver and
         * rate control, it might still start new aggregation
         * sessions -- block that to make sure the tear-down
         * will be sufficient.
         */
        set_sta_flag(sta, WLAN_STA_BLOCK_BA);
        ieee80211_sta_tear_down_BA_sessions(sta, AGG_STOP_DESTROY_STA);

        /*
         * Before removing the station from the driver there might be pending
         * rx frames on RSS queues sent prior to the disassociation - wait for
         * all such frames to be processed.
         */
        drv_sync_rx_queues(local, sta);

        for (i = 0; i < ARRAY_SIZE(sta->link); i++) {
                struct link_sta_info *link_sta;

                if (!(sta->sta.valid_links & BIT(i)))
                        continue;

                link_sta = rcu_dereference_protected(sta->link[i],
                                                     lockdep_is_held(&local->hw.wiphy->mtx));

                link_sta_info_hash_del(local, link_sta);
        }

        ret = sta_info_hash_del(local, sta);
        if (WARN_ON(ret))
                return ret;

        /*
         * for TDLS peers, make sure to return to the base channel before
         * removal.
         */
        if (test_sta_flag(sta, WLAN_STA_TDLS_OFF_CHANNEL)) {
                drv_tdls_cancel_channel_switch(local, sdata, &sta->sta);
                clear_sta_flag(sta, WLAN_STA_TDLS_OFF_CHANNEL);
        }

        list_del_rcu(&sta->list);
        sta->removed = true;

        if (sta->uploaded)
                drv_sta_pre_rcu_remove(local, sta->sdata, sta);

        if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN &&
            rcu_access_pointer(sdata->u.vlan.sta) == sta)
                RCU_INIT_POINTER(sdata->u.vlan.sta, NULL);

        return 0;
}

static int _sta_info_move_state(struct sta_info *sta,
                                enum ieee80211_sta_state new_state,
                                bool recalc)
{
        struct ieee80211_local *local = sta->local;

        might_sleep();

        if (sta->sta_state == new_state)
                return 0;

        /* check allowed transitions first */

        switch (new_state) {
        case IEEE80211_STA_NONE:
                if (sta->sta_state != IEEE80211_STA_AUTH)
                        return -EINVAL;
                break;
        case IEEE80211_STA_AUTH:
                if (sta->sta_state != IEEE80211_STA_NONE &&
                    sta->sta_state != IEEE80211_STA_ASSOC)
                        return -EINVAL;
                break;
        case IEEE80211_STA_ASSOC:
                if (sta->sta_state != IEEE80211_STA_AUTH &&
                    sta->sta_state != IEEE80211_STA_AUTHORIZED)
                        return -EINVAL;
                break;
        case IEEE80211_STA_AUTHORIZED:
                if (sta->sta_state != IEEE80211_STA_ASSOC)
                        return -EINVAL;
                break;
        default:
                WARN(1, "invalid state %d", new_state);
                return -EINVAL;
        }

        sta_dbg(sta->sdata, "moving STA %pM to state %d\n",
                sta->sta.addr, new_state);

        /* notify the driver before the actual changes so it can
         * fail the transition
         */
        if (test_sta_flag(sta, WLAN_STA_INSERTED)) {
                int err = drv_sta_state(sta->local, sta->sdata, sta,
                                        sta->sta_state, new_state);
                if (err)
                        return err;
        }

        /* reflect the change in all state variables */

        switch (new_state) {
        case IEEE80211_STA_NONE:
                if (sta->sta_state == IEEE80211_STA_AUTH)
                        clear_bit(WLAN_STA_AUTH, &sta->_flags);
                break;
        case IEEE80211_STA_AUTH:
                if (sta->sta_state == IEEE80211_STA_NONE) {
                        set_bit(WLAN_STA_AUTH, &sta->_flags);
                } else if (sta->sta_state == IEEE80211_STA_ASSOC) {
                        clear_bit(WLAN_STA_ASSOC, &sta->_flags);
                        if (recalc) {
                                ieee80211_recalc_min_chandef(sta->sdata, -1);
                                if (!sta->sta.support_p2p_ps)
                                        ieee80211_recalc_p2p_go_ps_allowed(sta->sdata);
                        }
                }
                break;
        case IEEE80211_STA_ASSOC:
                if (sta->sta_state == IEEE80211_STA_AUTH) {
                        set_bit(WLAN_STA_ASSOC, &sta->_flags);
                        sta->assoc_at = ktime_get_boottime_ns();
                        if (recalc) {
                                ieee80211_recalc_min_chandef(sta->sdata, -1);
                                if (!sta->sta.support_p2p_ps)
                                        ieee80211_recalc_p2p_go_ps_allowed(sta->sdata);
                        }
                } else if (sta->sta_state == IEEE80211_STA_AUTHORIZED) {
                        ieee80211_vif_dec_num_mcast(sta->sdata);
                        clear_bit(WLAN_STA_AUTHORIZED, &sta->_flags);

                        /*
                         * If we have encryption offload, flush (station) queues
                         * (after ensuring concurrent TX completed) so we won't
                         * transmit anything later unencrypted if/when keys are
                         * also removed, which might otherwise happen depending
                         * on how the hardware offload works.
                         */
                        if (local->ops->set_key) {
                                synchronize_net();
                                if (local->ops->flush_sta)
                                        drv_flush_sta(local, sta->sdata, sta);
                                else
                                        ieee80211_flush_queues(local,
                                                               sta->sdata,
                                                               false);
                        }

                        ieee80211_clear_fast_xmit(sta);
                        ieee80211_clear_fast_rx(sta);
                }
                break;
        case IEEE80211_STA_AUTHORIZED:
                if (sta->sta_state == IEEE80211_STA_ASSOC) {
                        ieee80211_vif_inc_num_mcast(sta->sdata);
                        set_bit(WLAN_STA_AUTHORIZED, &sta->_flags);
                        ieee80211_check_fast_xmit(sta);
                        ieee80211_check_fast_rx(sta);
                }
                if (sta->sdata->vif.type == NL80211_IFTYPE_AP_VLAN ||
                    sta->sdata->vif.type == NL80211_IFTYPE_AP)
                        cfg80211_send_layer2_update(sta->sdata->dev,
                                                    sta->sta.addr);
                break;
        default:
                break;
        }

        sta->sta_state = new_state;

        return 0;
}

int sta_info_move_state(struct sta_info *sta,
                        enum ieee80211_sta_state new_state)
{
        return _sta_info_move_state(sta, new_state, true);
}

static void __sta_info_destroy_part2(struct sta_info *sta, bool recalc)
{
        struct ieee80211_local *local = sta->local;
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct station_info *sinfo;
        int ret;

        /*
         * NOTE: This assumes at least synchronize_net() was done
         *         after _part1 and before _part2!
         */

        /*
         * There's a potential race in _part1 where we set WLAN_STA_BLOCK_BA
         * but someone might have just gotten past a check, and not yet into
         * queuing the work/creating the data/etc.
         *
         * Do another round of destruction so that the worker is certainly
         * canceled before we later free the station.
         *
         * Since this is after synchronize_rcu()/synchronize_net() we're now
         * certain that nobody can actually hold a reference to the STA and
         * be calling e.g. ieee80211_start_tx_ba_session().
         */
        ieee80211_sta_tear_down_BA_sessions(sta, AGG_STOP_DESTROY_STA);

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (sta->sta_state == IEEE80211_STA_AUTHORIZED) {
                ret = _sta_info_move_state(sta, IEEE80211_STA_ASSOC, recalc);
                WARN_ON_ONCE(ret);
        }

        /* now keys can no longer be reached */
        ieee80211_free_sta_keys(local, sta);

        /* disable TIM bit - last chance to tell driver */
        __sta_info_recalc_tim(sta, true);

        sta->dead = true;

        local->num_sta--;
        local->sta_generation++;

        while (sta->sta_state > IEEE80211_STA_NONE) {
                ret = _sta_info_move_state(sta, sta->sta_state - 1, recalc);
                if (ret) {
                        WARN_ON_ONCE(1);
                        break;
                }
        }

        if (sta->uploaded) {
                ret = drv_sta_state(local, sdata, sta, IEEE80211_STA_NONE,
                                    IEEE80211_STA_NOTEXIST);
                WARN_ON_ONCE(ret != 0);
        }

        sta_dbg(sdata, "Removed STA %pM\n", sta->sta.addr);

        sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL);
        if (sinfo)
                sta_set_sinfo(sta, sinfo, true);
        cfg80211_del_sta_sinfo(sdata->dev, sta->sta.addr, sinfo, GFP_KERNEL);
        kfree(sinfo);

        ieee80211_sta_debugfs_remove(sta);

        ieee80211_destroy_frag_cache(&sta->frags);

        cleanup_single_sta(sta);
}

int __must_check __sta_info_destroy(struct sta_info *sta)
{
        int err = __sta_info_destroy_part1(sta);

        if (err)
                return err;

        synchronize_net();

        __sta_info_destroy_part2(sta, true);

        return 0;
}

int sta_info_destroy_addr(struct ieee80211_sub_if_data *sdata, const u8 *addr)
{
        struct sta_info *sta;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        sta = sta_info_get(sdata, addr);
        return __sta_info_destroy(sta);
}

int sta_info_destroy_addr_bss(struct ieee80211_sub_if_data *sdata,
                              const u8 *addr)
{
        struct sta_info *sta;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        sta = sta_info_get_bss(sdata, addr);
        return __sta_info_destroy(sta);
}

static void sta_info_cleanup(struct timer_list *t)
{
        struct ieee80211_local *local = from_timer(local, t, sta_cleanup);
        struct sta_info *sta;
        bool timer_needed = false;

        rcu_read_lock();
        list_for_each_entry_rcu(sta, &local->sta_list, list)
                if (sta_info_cleanup_expire_buffered(local, sta))
                        timer_needed = true;
        rcu_read_unlock();

        if (local->quiescing)
                return;

        if (!timer_needed)
                return;

        mod_timer(&local->sta_cleanup,
                  round_jiffies(jiffies + STA_INFO_CLEANUP_INTERVAL));
}

int sta_info_init(struct ieee80211_local *local)
{
        int err;

        err = rhltable_init(&local->sta_hash, &sta_rht_params);
        if (err)
                return err;

        err = rhltable_init(&local->link_sta_hash, &link_sta_rht_params);
        if (err) {
                rhltable_destroy(&local->sta_hash);
                return err;
        }

        spin_lock_init(&local->tim_lock);
        INIT_LIST_HEAD(&local->sta_list);

        timer_setup(&local->sta_cleanup, sta_info_cleanup, 0);
        return 0;
}

void sta_info_stop(struct ieee80211_local *local)
{
        del_timer_sync(&local->sta_cleanup);
        rhltable_destroy(&local->sta_hash);
        rhltable_destroy(&local->link_sta_hash);
}


int __sta_info_flush(struct ieee80211_sub_if_data *sdata, bool vlans,
                     int link_id)
{
        struct ieee80211_local *local = sdata->local;
        struct sta_info *sta, *tmp;
        LIST_HEAD(free_list);
        int ret = 0;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        WARN_ON(vlans && sdata->vif.type != NL80211_IFTYPE_AP);
        WARN_ON(vlans && !sdata->bss);

        list_for_each_entry_safe(sta, tmp, &local->sta_list, list) {
                if (sdata != sta->sdata &&
                    (!vlans || sdata->bss != sta->sdata->bss))
                        continue;

                if (link_id >= 0 && sta->sta.valid_links &&
                    !(sta->sta.valid_links & BIT(link_id)))
                        continue;

                if (!WARN_ON(__sta_info_destroy_part1(sta)))
                        list_add(&sta->free_list, &free_list);

                ret++;
        }

        if (!list_empty(&free_list)) {
                bool support_p2p_ps = true;

                synchronize_net();
                list_for_each_entry_safe(sta, tmp, &free_list, free_list) {
                        if (!sta->sta.support_p2p_ps)
                                support_p2p_ps = false;
                        __sta_info_destroy_part2(sta, false);
                }

                ieee80211_recalc_min_chandef(sdata, -1);
                if (!support_p2p_ps)
                        ieee80211_recalc_p2p_go_ps_allowed(sdata);
        }

        return ret;
}

void ieee80211_sta_expire(struct ieee80211_sub_if_data *sdata,
                          unsigned long exp_time)
{
        struct ieee80211_local *local = sdata->local;
        struct sta_info *sta, *tmp;

        lockdep_assert_wiphy(local->hw.wiphy);

        list_for_each_entry_safe(sta, tmp, &local->sta_list, list) {
                unsigned long last_active = ieee80211_sta_last_active(sta);

                if (sdata != sta->sdata)
                        continue;

                if (time_is_before_jiffies(last_active + exp_time)) {
                        sta_dbg(sta->sdata, "expiring inactive STA %pM\n",
                                sta->sta.addr);

                        if (ieee80211_vif_is_mesh(&sdata->vif) &&
                            test_sta_flag(sta, WLAN_STA_PS_STA))
                                atomic_dec(&sdata->u.mesh.ps.num_sta_ps);

                        WARN_ON(__sta_info_destroy(sta));
                }
        }
}

struct ieee80211_sta *ieee80211_find_sta_by_ifaddr(struct ieee80211_hw *hw,
                                                   const u8 *addr,
                                                   const u8 *localaddr)
{
        struct ieee80211_local *local = hw_to_local(hw);
        struct rhlist_head *tmp;
        struct sta_info *sta;

        /*
         * Just return a random station if localaddr is NULL
         * ... first in list.
         */
        for_each_sta_info(local, addr, sta, tmp) {
                if (localaddr &&
                    !ether_addr_equal(sta->sdata->vif.addr, localaddr))
                        continue;
                if (!sta->uploaded)
                        return NULL;
                return &sta->sta;
        }

        return NULL;
}
EXPORT_SYMBOL_GPL(ieee80211_find_sta_by_ifaddr);

struct ieee80211_sta *ieee80211_find_sta(struct ieee80211_vif *vif,
                                         const u8 *addr)
{
        struct sta_info *sta;

        if (!vif)
                return NULL;

        sta = sta_info_get_bss(vif_to_sdata(vif), addr);
        if (!sta)
                return NULL;

        if (!sta->uploaded)
                return NULL;

        return &sta->sta;
}
EXPORT_SYMBOL(ieee80211_find_sta);

/* powersave support code */
void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        struct sk_buff_head pending;
        int filtered = 0, buffered = 0, ac, i;
        unsigned long flags;
        struct ps_data *ps;

        if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
                sdata = container_of(sdata->bss, struct ieee80211_sub_if_data,
                                     u.ap);

        if (sdata->vif.type == NL80211_IFTYPE_AP)
                ps = &sdata->bss->ps;
        else if (ieee80211_vif_is_mesh(&sdata->vif))
                ps = &sdata->u.mesh.ps;
        else
                return;

        clear_sta_flag(sta, WLAN_STA_SP);

        BUILD_BUG_ON(BITS_TO_LONGS(IEEE80211_NUM_TIDS) > 1);
        sta->driver_buffered_tids = 0;
        sta->txq_buffered_tids = 0;

        if (!ieee80211_hw_check(&local->hw, AP_LINK_PS))
                drv_sta_notify(local, sdata, STA_NOTIFY_AWAKE, &sta->sta);

        for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
                if (!sta->sta.txq[i] || !txq_has_queue(sta->sta.txq[i]))
                        continue;

                schedule_and_wake_txq(local, to_txq_info(sta->sta.txq[i]));
        }

        skb_queue_head_init(&pending);

        /* sync with ieee80211_tx_h_unicast_ps_buf */
        spin_lock_bh(&sta->ps_lock);
        /* Send all buffered frames to the station */
        for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
                int count = skb_queue_len(&pending), tmp;

                spin_lock_irqsave(&sta->tx_filtered[ac].lock, flags);
                skb_queue_splice_tail_init(&sta->tx_filtered[ac], &pending);
                spin_unlock_irqrestore(&sta->tx_filtered[ac].lock, flags);
                tmp = skb_queue_len(&pending);
                filtered += tmp - count;
                count = tmp;

                spin_lock_irqsave(&sta->ps_tx_buf[ac].lock, flags);
                skb_queue_splice_tail_init(&sta->ps_tx_buf[ac], &pending);
                spin_unlock_irqrestore(&sta->ps_tx_buf[ac].lock, flags);
                tmp = skb_queue_len(&pending);
                buffered += tmp - count;
        }

        ieee80211_add_pending_skbs(local, &pending);

        /* now we're no longer in the deliver code */
        clear_sta_flag(sta, WLAN_STA_PS_DELIVER);

        /* The station might have polled and then woken up before we responded,
         * so clear these flags now to avoid them sticking around.
         */
        clear_sta_flag(sta, WLAN_STA_PSPOLL);
        clear_sta_flag(sta, WLAN_STA_UAPSD);
        spin_unlock_bh(&sta->ps_lock);

        atomic_dec(&ps->num_sta_ps);

        local->total_ps_buffered -= buffered;

        sta_info_recalc_tim(sta);

        ps_dbg(sdata,
               "STA %pM aid %d sending %d filtered/%d PS frames since STA woke up\n",
               sta->sta.addr, sta->sta.aid, filtered, buffered);

        ieee80211_check_fast_xmit(sta);
}

static void ieee80211_send_null_response(struct sta_info *sta, int tid,
                                         enum ieee80211_frame_release_type reason,
                                         bool call_driver, bool more_data)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_qos_hdr *nullfunc;
        struct sk_buff *skb;
        int size = sizeof(*nullfunc);
        __le16 fc;
        bool qos = sta->sta.wme;
        struct ieee80211_tx_info *info;
        struct ieee80211_chanctx_conf *chanctx_conf;

        if (qos) {
                fc = cpu_to_le16(IEEE80211_FTYPE_DATA |
                                 IEEE80211_STYPE_QOS_NULLFUNC |
                                 IEEE80211_FCTL_FROMDS);
        } else {
                size -= 2;
                fc = cpu_to_le16(IEEE80211_FTYPE_DATA |
                                 IEEE80211_STYPE_NULLFUNC |
                                 IEEE80211_FCTL_FROMDS);
        }

        skb = dev_alloc_skb(local->hw.extra_tx_headroom + size);
        if (!skb)
                return;

        skb_reserve(skb, local->hw.extra_tx_headroom);

        nullfunc = skb_put(skb, size);
        nullfunc->frame_control = fc;
        nullfunc->duration_id = 0;
        memcpy(nullfunc->addr1, sta->sta.addr, ETH_ALEN);
        memcpy(nullfunc->addr2, sdata->vif.addr, ETH_ALEN);
        memcpy(nullfunc->addr3, sdata->vif.addr, ETH_ALEN);
        nullfunc->seq_ctrl = 0;

        skb->priority = tid;
        skb_set_queue_mapping(skb, ieee802_1d_to_ac[tid]);
        if (qos) {
                nullfunc->qos_ctrl = cpu_to_le16(tid);

                if (reason == IEEE80211_FRAME_RELEASE_UAPSD) {
                        nullfunc->qos_ctrl |=
                                cpu_to_le16(IEEE80211_QOS_CTL_EOSP);
                        if (more_data)
                                nullfunc->frame_control |=
                                        cpu_to_le16(IEEE80211_FCTL_MOREDATA);
                }
        }

        info = IEEE80211_SKB_CB(skb);

        /*
         * Tell TX path to send this frame even though the
         * STA may still remain is PS mode after this frame
         * exchange. Also set EOSP to indicate this packet
         * ends the poll/service period.
         */
        info->flags |= IEEE80211_TX_CTL_NO_PS_BUFFER |
                       IEEE80211_TX_STATUS_EOSP |
                       IEEE80211_TX_CTL_REQ_TX_STATUS;

        info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE;

        if (call_driver)
                drv_allow_buffered_frames(local, sta, BIT(tid), 1,
                                          reason, false);

        skb->dev = sdata->dev;

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
        if (WARN_ON(!chanctx_conf)) {
                rcu_read_unlock();
                kfree_skb(skb);
                return;
        }

        info->band = chanctx_conf->def.chan->band;
        ieee80211_xmit(sdata, sta, skb);
        rcu_read_unlock();
}

static int find_highest_prio_tid(unsigned long tids)
{
        /* lower 3 TIDs aren't ordered perfectly */
        if (tids & 0xF8)
                return fls(tids) - 1;
        /* TID 0 is BE just like TID 3 */
        if (tids & BIT(0))
                return 0;
        return fls(tids) - 1;
}

/* Indicates if the MORE_DATA bit should be set in the last
 * frame obtained by ieee80211_sta_ps_get_frames.
 * Note that driver_release_tids is relevant only if
 * reason = IEEE80211_FRAME_RELEASE_PSPOLL
 */
static bool
ieee80211_sta_ps_more_data(struct sta_info *sta, u8 ignored_acs,
                           enum ieee80211_frame_release_type reason,
                           unsigned long driver_release_tids)
{
        int ac;

        /* If the driver has data on more than one TID then
         * certainly there's more data if we release just a
         * single frame now (from a single TID). This will
         * only happen for PS-Poll.
         */
        if (reason == IEEE80211_FRAME_RELEASE_PSPOLL &&
            hweight16(driver_release_tids) > 1)
                return true;

        for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
                if (ignored_acs & ieee80211_ac_to_qos_mask[ac])
                        continue;

                if (!skb_queue_empty(&sta->tx_filtered[ac]) ||
                    !skb_queue_empty(&sta->ps_tx_buf[ac]))
                        return true;
        }

        return false;
}

static void
ieee80211_sta_ps_get_frames(struct sta_info *sta, int n_frames, u8 ignored_acs,
                            enum ieee80211_frame_release_type reason,
                            struct sk_buff_head *frames,
                            unsigned long *driver_release_tids)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        int ac;

        /* Get response frame(s) and more data bit for the last one. */
        for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
                unsigned long tids;

                if (ignored_acs & ieee80211_ac_to_qos_mask[ac])
                        continue;

                tids = ieee80211_tids_for_ac(ac);

                /* if we already have frames from software, then we can't also
                 * release from hardware queues
                 */
                if (skb_queue_empty(frames)) {
                        *driver_release_tids |=
                                sta->driver_buffered_tids & tids;
                        *driver_release_tids |= sta->txq_buffered_tids & tids;
                }

                if (!*driver_release_tids) {
                        struct sk_buff *skb;

                        while (n_frames > 0) {
                                skb = skb_dequeue(&sta->tx_filtered[ac]);
                                if (!skb) {
                                        skb = skb_dequeue(
                                                &sta->ps_tx_buf[ac]);
                                        if (skb)
                                                local->total_ps_buffered--;
                                }
                                if (!skb)
                                        break;
                                n_frames--;
                                __skb_queue_tail(frames, skb);
                        }
                }

                /* If we have more frames buffered on this AC, then abort the
                 * loop since we can't send more data from other ACs before
                 * the buffered frames from this.
                 */
                if (!skb_queue_empty(&sta->tx_filtered[ac]) ||
                    !skb_queue_empty(&sta->ps_tx_buf[ac]))
                        break;
        }
}

static void
ieee80211_sta_ps_deliver_response(struct sta_info *sta,
                                  int n_frames, u8 ignored_acs,
                                  enum ieee80211_frame_release_type reason)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        unsigned long driver_release_tids = 0;
        struct sk_buff_head frames;
        bool more_data;

        /* Service or PS-Poll period starts */
        set_sta_flag(sta, WLAN_STA_SP);

        __skb_queue_head_init(&frames);

        ieee80211_sta_ps_get_frames(sta, n_frames, ignored_acs, reason,
                                    &frames, &driver_release_tids);

        more_data = ieee80211_sta_ps_more_data(sta, ignored_acs, reason, driver_release_tids);

        if (driver_release_tids && reason == IEEE80211_FRAME_RELEASE_PSPOLL)
                driver_release_tids =
                        BIT(find_highest_prio_tid(driver_release_tids));

        if (skb_queue_empty(&frames) && !driver_release_tids) {
                int tid, ac;

                /*
                 * For PS-Poll, this can only happen due to a race condition
                 * when we set the TIM bit and the station notices it, but
                 * before it can poll for the frame we expire it.
                 *
                 * For uAPSD, this is said in the standard (11.2.1.5 h):
                 *        At each unscheduled SP for a non-AP STA, the AP shall
                 *        attempt to transmit at least one MSDU or MMPDU, but no
                 *        more than the value specified in the Max SP Length field
                 *        in the QoS Capability element from delivery-enabled ACs,
                 *        that are destined for the non-AP STA.
                 *
                 * Since we have no other MSDU/MMPDU, transmit a QoS null frame.
                 */

                /* This will evaluate to 1, 3, 5 or 7. */
                for (ac = IEEE80211_AC_VO; ac < IEEE80211_NUM_ACS; ac++)
                        if (!(ignored_acs & ieee80211_ac_to_qos_mask[ac]))
                                break;
                tid = 7 - 2 * ac;

                ieee80211_send_null_response(sta, tid, reason, true, false);
        } else if (!driver_release_tids) {
                struct sk_buff_head pending;
                struct sk_buff *skb;
                int num = 0;
                u16 tids = 0;
                bool need_null = false;

                skb_queue_head_init(&pending);

                while ((skb = __skb_dequeue(&frames))) {
                        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
                        struct ieee80211_hdr *hdr = (void *) skb->data;
                        u8 *qoshdr = NULL;

                        num++;

                        /*
                         * Tell TX path to send this frame even though the
                         * STA may still remain is PS mode after this frame
                         * exchange.
                         */
                        info->flags |= IEEE80211_TX_CTL_NO_PS_BUFFER;
                        info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE;

                        /*
                         * Use MoreData flag to indicate whether there are
                         * more buffered frames for this STA
                         */
                        if (more_data || !skb_queue_empty(&frames))
                                hdr->frame_control |=
                                        cpu_to_le16(IEEE80211_FCTL_MOREDATA);
                        else
                                hdr->frame_control &=
                                        cpu_to_le16(~IEEE80211_FCTL_MOREDATA);

                        if (ieee80211_is_data_qos(hdr->frame_control) ||
                            ieee80211_is_qos_nullfunc(hdr->frame_control))
                                qoshdr = ieee80211_get_qos_ctl(hdr);

                        tids |= BIT(skb->priority);

                        __skb_queue_tail(&pending, skb);

                        /* end service period after last frame or add one */
                        if (!skb_queue_empty(&frames))
                                continue;

                        if (reason != IEEE80211_FRAME_RELEASE_UAPSD) {
                                /* for PS-Poll, there's only one frame */
                                info->flags |= IEEE80211_TX_STATUS_EOSP |
                                               IEEE80211_TX_CTL_REQ_TX_STATUS;
                                break;
                        }

                        /* For uAPSD, things are a bit more complicated. If the
                         * last frame has a QoS header (i.e. is a QoS-data or
                         * QoS-nulldata frame) then just set the EOSP bit there
                         * and be done.
                         * If the frame doesn't have a QoS header (which means
                         * it should be a bufferable MMPDU) then we can't set
                         * the EOSP bit in the QoS header; add a QoS-nulldata
                         * frame to the list to send it after the MMPDU.
                         *
                         * Note that this code is only in the mac80211-release
                         * code path, we assume that the driver will not buffer
                         * anything but QoS-data frames, or if it does, will
                         * create the QoS-nulldata frame by itself if needed.
                         *
                         * Cf. 802.11-2012 10.2.1.10 (c).
                         */
                        if (qoshdr) {
                                *qoshdr |= IEEE80211_QOS_CTL_EOSP;

                                info->flags |= IEEE80211_TX_STATUS_EOSP |
                                               IEEE80211_TX_CTL_REQ_TX_STATUS;
                        } else {
                                /* The standard isn't completely clear on this
                                 * as it says the more-data bit should be set
                                 * if there are more BUs. The QoS-Null frame
                                 * we're about to send isn't buffered yet, we
                                 * only create it below, but let's pretend it
                                 * was buffered just in case some clients only
                                 * expect more-data=0 when eosp=1.
                                 */
                                hdr->frame_control |=
                                        cpu_to_le16(IEEE80211_FCTL_MOREDATA);
                                need_null = true;
                                num++;
                        }
                        break;
                }

                drv_allow_buffered_frames(local, sta, tids, num,
                                          reason, more_data);

                ieee80211_add_pending_skbs(local, &pending);

                if (need_null)
                        ieee80211_send_null_response(
                                sta, find_highest_prio_tid(tids),
                                reason, false, false);

                sta_info_recalc_tim(sta);
        } else {
                int tid;

                /*
                 * We need to release a frame that is buffered somewhere in the
                 * driver ... it'll have to handle that.
                 * Note that the driver also has to check the number of frames
                 * on the TIDs we're releasing from - if there are more than
                 * n_frames it has to set the more-data bit (if we didn't ask
                 * it to set it anyway due to other buffered frames); if there
                 * are fewer than n_frames it has to make sure to adjust that
                 * to allow the service period to end properly.
                 */
                drv_release_buffered_frames(local, sta, driver_release_tids,
                                            n_frames, reason, more_data);

                /*
                 * Note that we don't recalculate the TIM bit here as it would
                 * most likely have no effect at all unless the driver told us
                 * that the TID(s) became empty before returning here from the
                 * release function.
                 * Either way, however, when the driver tells us that the TID(s)
                 * became empty or we find that a txq became empty, we'll do the
                 * TIM recalculation.
                 */

                for (tid = 0; tid < ARRAY_SIZE(sta->sta.txq); tid++) {
                        if (!sta->sta.txq[tid] ||
                            !(driver_release_tids & BIT(tid)) ||
                            txq_has_queue(sta->sta.txq[tid]))
                                continue;

                        sta_info_recalc_tim(sta);
                        break;
                }
        }
}

void ieee80211_sta_ps_deliver_poll_response(struct sta_info *sta)
{
        u8 ignore_for_response = sta->sta.uapsd_queues;

        /*
         * If all ACs are delivery-enabled then we should reply
         * from any of them, if only some are enabled we reply
         * only from the non-enabled ones.
         */
        if (ignore_for_response == BIT(IEEE80211_NUM_ACS) - 1)
                ignore_for_response = 0;

        ieee80211_sta_ps_deliver_response(sta, 1, ignore_for_response,
                                          IEEE80211_FRAME_RELEASE_PSPOLL);
}

void ieee80211_sta_ps_deliver_uapsd(struct sta_info *sta)
{
        int n_frames = sta->sta.max_sp;
        u8 delivery_enabled = sta->sta.uapsd_queues;

        /*
         * If we ever grow support for TSPEC this might happen if
         * the TSPEC update from hostapd comes in between a trigger
         * frame setting WLAN_STA_UAPSD in the RX path and this
         * actually getting called.
         */
        if (!delivery_enabled)
                return;

        switch (sta->sta.max_sp) {
        case 1:
                n_frames = 2;
                break;
        case 2:
                n_frames = 4;
                break;
        case 3:
                n_frames = 6;
                break;
        case 0:
                /* XXX: what is a good value? */
                n_frames = 128;
                break;
        }

        ieee80211_sta_ps_deliver_response(sta, n_frames, ~delivery_enabled,
                                          IEEE80211_FRAME_RELEASE_UAPSD);
}

void ieee80211_sta_block_awake(struct ieee80211_hw *hw,
                               struct ieee80211_sta *pubsta, bool block)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);

        trace_api_sta_block_awake(sta->local, pubsta, block);

        if (block) {
                set_sta_flag(sta, WLAN_STA_PS_DRIVER);
                ieee80211_clear_fast_xmit(sta);
                return;
        }

        if (!test_sta_flag(sta, WLAN_STA_PS_DRIVER))
                return;

        if (!test_sta_flag(sta, WLAN_STA_PS_STA)) {
                set_sta_flag(sta, WLAN_STA_PS_DELIVER);
                clear_sta_flag(sta, WLAN_STA_PS_DRIVER);
                ieee80211_queue_work(hw, &sta->drv_deliver_wk);
        } else if (test_sta_flag(sta, WLAN_STA_PSPOLL) ||
                   test_sta_flag(sta, WLAN_STA_UAPSD)) {
                /* must be asleep in this case */
                clear_sta_flag(sta, WLAN_STA_PS_DRIVER);
                ieee80211_queue_work(hw, &sta->drv_deliver_wk);
        } else {
                clear_sta_flag(sta, WLAN_STA_PS_DRIVER);
                ieee80211_check_fast_xmit(sta);
        }
}
EXPORT_SYMBOL(ieee80211_sta_block_awake);

void ieee80211_sta_eosp(struct ieee80211_sta *pubsta)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
        struct ieee80211_local *local = sta->local;

        trace_api_eosp(local, pubsta);

        clear_sta_flag(sta, WLAN_STA_SP);
}
EXPORT_SYMBOL(ieee80211_sta_eosp);

void ieee80211_send_eosp_nullfunc(struct ieee80211_sta *pubsta, int tid)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
        enum ieee80211_frame_release_type reason;
        bool more_data;

        trace_api_send_eosp_nullfunc(sta->local, pubsta, tid);

        reason = IEEE80211_FRAME_RELEASE_UAPSD;
        more_data = ieee80211_sta_ps_more_data(sta, ~sta->sta.uapsd_queues,
                                               reason, 0);

        ieee80211_send_null_response(sta, tid, reason, false, more_data);
}
EXPORT_SYMBOL(ieee80211_send_eosp_nullfunc);

void ieee80211_sta_set_buffered(struct ieee80211_sta *pubsta,
                                u8 tid, bool buffered)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);

        if (WARN_ON(tid >= IEEE80211_NUM_TIDS))
                return;

        trace_api_sta_set_buffered(sta->local, pubsta, tid, buffered);

        if (buffered)
                set_bit(tid, &sta->driver_buffered_tids);
        else
                clear_bit(tid, &sta->driver_buffered_tids);

        sta_info_recalc_tim(sta);
}
EXPORT_SYMBOL(ieee80211_sta_set_buffered);

void ieee80211_sta_register_airtime(struct ieee80211_sta *pubsta, u8 tid,
                                    u32 tx_airtime, u32 rx_airtime)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
        struct ieee80211_local *local = sta->sdata->local;
        u8 ac = ieee80211_ac_from_tid(tid);
        u32 airtime = 0;

        if (sta->local->airtime_flags & AIRTIME_USE_TX)
                airtime += tx_airtime;
        if (sta->local->airtime_flags & AIRTIME_USE_RX)
                airtime += rx_airtime;

        spin_lock_bh(&local->active_txq_lock[ac]);
        sta->airtime[ac].tx_airtime += tx_airtime;
        sta->airtime[ac].rx_airtime += rx_airtime;

        if (ieee80211_sta_keep_active(sta, ac))
                sta->airtime[ac].deficit -= airtime;

        spin_unlock_bh(&local->active_txq_lock[ac]);
}
EXPORT_SYMBOL(ieee80211_sta_register_airtime);

void __ieee80211_sta_recalc_aggregates(struct sta_info *sta, u16 active_links)
{
        bool first = true;
        int link_id;

        if (!sta->sta.valid_links || !sta->sta.mlo) {
                sta->sta.cur = &sta->sta.deflink.agg;
                return;
        }

        rcu_read_lock();
        for (link_id = 0; link_id < ARRAY_SIZE((sta)->link); link_id++) {
                struct ieee80211_link_sta *link_sta;
                int i;

                if (!(active_links & BIT(link_id)))
                        continue;

                link_sta = rcu_dereference(sta->sta.link[link_id]);
                if (!link_sta)
                        continue;

                if (first) {
                        sta->cur = sta->sta.deflink.agg;
                        first = false;
                        continue;
                }

                sta->cur.max_amsdu_len =
                        min(sta->cur.max_amsdu_len,
                            link_sta->agg.max_amsdu_len);
                sta->cur.max_rc_amsdu_len =
                        min(sta->cur.max_rc_amsdu_len,
                            link_sta->agg.max_rc_amsdu_len);

                for (i = 0; i < ARRAY_SIZE(sta->cur.max_tid_amsdu_len); i++)
                        sta->cur.max_tid_amsdu_len[i] =
                                min(sta->cur.max_tid_amsdu_len[i],
                                    link_sta->agg.max_tid_amsdu_len[i]);
        }
        rcu_read_unlock();

        sta->sta.cur = &sta->cur;
}

void ieee80211_sta_recalc_aggregates(struct ieee80211_sta *pubsta)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);

        __ieee80211_sta_recalc_aggregates(sta, sta->sdata->vif.active_links);
}
EXPORT_SYMBOL(ieee80211_sta_recalc_aggregates);

void ieee80211_sta_update_pending_airtime(struct ieee80211_local *local,
                                          struct sta_info *sta, u8 ac,
                                          u16 tx_airtime, bool tx_completed)
{
        int tx_pending;

        if (!wiphy_ext_feature_isset(local->hw.wiphy, NL80211_EXT_FEATURE_AQL))
                return;

        if (!tx_completed) {
                if (sta)
                        atomic_add(tx_airtime,
                                   &sta->airtime[ac].aql_tx_pending);

                atomic_add(tx_airtime, &local->aql_total_pending_airtime);
                atomic_add(tx_airtime, &local->aql_ac_pending_airtime[ac]);
                return;
        }

        if (sta) {
                tx_pending = atomic_sub_return(tx_airtime,
                                               &sta->airtime[ac].aql_tx_pending);
                if (tx_pending < 0)
                        atomic_cmpxchg(&sta->airtime[ac].aql_tx_pending,
                                       tx_pending, 0);
        }

        atomic_sub(tx_airtime, &local->aql_total_pending_airtime);
        tx_pending = atomic_sub_return(tx_airtime,
                                       &local->aql_ac_pending_airtime[ac]);
        if (WARN_ONCE(tx_pending < 0,
                      "Device %s AC %d pending airtime underflow: %u, %u",
                      wiphy_name(local->hw.wiphy), ac, tx_pending,
                      tx_airtime)) {
                atomic_cmpxchg(&local->aql_ac_pending_airtime[ac],
                               tx_pending, 0);
                atomic_sub(tx_pending, &local->aql_total_pending_airtime);
        }
}

static struct ieee80211_sta_rx_stats *
sta_get_last_rx_stats(struct sta_info *sta)
{
        struct ieee80211_sta_rx_stats *stats = &sta->deflink.rx_stats;
        int cpu;

        if (!sta->deflink.pcpu_rx_stats)
                return stats;

        for_each_possible_cpu(cpu) {
                struct ieee80211_sta_rx_stats *cpustats;

                cpustats = per_cpu_ptr(sta->deflink.pcpu_rx_stats, cpu);

                if (time_after(cpustats->last_rx, stats->last_rx))
                        stats = cpustats;
        }

        return stats;
}

static void sta_stats_decode_rate(struct ieee80211_local *local, u32 rate,
                                  struct rate_info *rinfo)
{
        rinfo->bw = STA_STATS_GET(BW, rate);

        switch (STA_STATS_GET(TYPE, rate)) {
        case STA_STATS_RATE_TYPE_VHT:
                rinfo->flags = RATE_INFO_FLAGS_VHT_MCS;
                rinfo->mcs = STA_STATS_GET(VHT_MCS, rate);
                rinfo->nss = STA_STATS_GET(VHT_NSS, rate);
                if (STA_STATS_GET(SGI, rate))
                        rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI;
                break;
        case STA_STATS_RATE_TYPE_HT:
                rinfo->flags = RATE_INFO_FLAGS_MCS;
                rinfo->mcs = STA_STATS_GET(HT_MCS, rate);
                if (STA_STATS_GET(SGI, rate))
                        rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI;
                break;
        case STA_STATS_RATE_TYPE_LEGACY: {
                struct ieee80211_supported_band *sband;
                u16 brate;
                unsigned int shift;
                int band = STA_STATS_GET(LEGACY_BAND, rate);
                int rate_idx = STA_STATS_GET(LEGACY_IDX, rate);

                sband = local->hw.wiphy->bands[band];

                if (WARN_ON_ONCE(!sband->bitrates))
                        break;

                brate = sband->bitrates[rate_idx].bitrate;
                if (rinfo->bw == RATE_INFO_BW_5)
                        shift = 2;
                else if (rinfo->bw == RATE_INFO_BW_10)
                        shift = 1;
                else
                        shift = 0;
                rinfo->legacy = DIV_ROUND_UP(brate, 1 << shift);
                break;
                }
        case STA_STATS_RATE_TYPE_HE:
                rinfo->flags = RATE_INFO_FLAGS_HE_MCS;
                rinfo->mcs = STA_STATS_GET(HE_MCS, rate);
                rinfo->nss = STA_STATS_GET(HE_NSS, rate);
                rinfo->he_gi = STA_STATS_GET(HE_GI, rate);
                rinfo->he_ru_alloc = STA_STATS_GET(HE_RU, rate);
                rinfo->he_dcm = STA_STATS_GET(HE_DCM, rate);
                break;
        case STA_STATS_RATE_TYPE_EHT:
                rinfo->flags = RATE_INFO_FLAGS_EHT_MCS;
                rinfo->mcs = STA_STATS_GET(EHT_MCS, rate);
                rinfo->nss = STA_STATS_GET(EHT_NSS, rate);
                rinfo->eht_gi = STA_STATS_GET(EHT_GI, rate);
                rinfo->eht_ru_alloc = STA_STATS_GET(EHT_RU, rate);
                break;
        }
}

static int sta_set_rate_info_rx(struct sta_info *sta, struct rate_info *rinfo)
{
        u32 rate = READ_ONCE(sta_get_last_rx_stats(sta)->last_rate);

        if (rate == STA_STATS_RATE_INVALID)
                return -EINVAL;

        sta_stats_decode_rate(sta->local, rate, rinfo);
        return 0;
}

static inline u64 sta_get_tidstats_msdu(struct ieee80211_sta_rx_stats *rxstats,
                                        int tid)
{
        unsigned int start;
        u64 value;

        do {
                start = u64_stats_fetch_begin(&rxstats->syncp);
                value = rxstats->msdu[tid];
        } while (u64_stats_fetch_retry(&rxstats->syncp, start));

        return value;
}

static void sta_set_tidstats(struct sta_info *sta,
                             struct cfg80211_tid_stats *tidstats,
                             int tid)
{
        struct ieee80211_local *local = sta->local;
        int cpu;

        if (!(tidstats->filled & BIT(NL80211_TID_STATS_RX_MSDU))) {
                tidstats->rx_msdu += sta_get_tidstats_msdu(&sta->deflink.rx_stats,
                                                           tid);

                if (sta->deflink.pcpu_rx_stats) {
                        for_each_possible_cpu(cpu) {
                                struct ieee80211_sta_rx_stats *cpurxs;

                                cpurxs = per_cpu_ptr(sta->deflink.pcpu_rx_stats,
                                                     cpu);
                                tidstats->rx_msdu +=
                                        sta_get_tidstats_msdu(cpurxs, tid);
                        }
                }

                tidstats->filled |= BIT(NL80211_TID_STATS_RX_MSDU);
        }

        if (!(tidstats->filled & BIT(NL80211_TID_STATS_TX_MSDU))) {
                tidstats->filled |= BIT(NL80211_TID_STATS_TX_MSDU);
                tidstats->tx_msdu = sta->deflink.tx_stats.msdu[tid];
        }

        if (!(tidstats->filled & BIT(NL80211_TID_STATS_TX_MSDU_RETRIES)) &&
            ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS)) {
                tidstats->filled |= BIT(NL80211_TID_STATS_TX_MSDU_RETRIES);
                tidstats->tx_msdu_retries = sta->deflink.status_stats.msdu_retries[tid];
        }

        if (!(tidstats->filled & BIT(NL80211_TID_STATS_TX_MSDU_FAILED)) &&
            ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS)) {
                tidstats->filled |= BIT(NL80211_TID_STATS_TX_MSDU_FAILED);
                tidstats->tx_msdu_failed = sta->deflink.status_stats.msdu_failed[tid];
        }

        if (tid < IEEE80211_NUM_TIDS) {
                spin_lock_bh(&local->fq.lock);
                rcu_read_lock();

                tidstats->filled |= BIT(NL80211_TID_STATS_TXQ_STATS);
                ieee80211_fill_txq_stats(&tidstats->txq_stats,
                                         to_txq_info(sta->sta.txq[tid]));

                rcu_read_unlock();
                spin_unlock_bh(&local->fq.lock);
        }
}

static inline u64 sta_get_stats_bytes(struct ieee80211_sta_rx_stats *rxstats)
{
        unsigned int start;
        u64 value;

        do {
                start = u64_stats_fetch_begin(&rxstats->syncp);
                value = rxstats->bytes;
        } while (u64_stats_fetch_retry(&rxstats->syncp, start));

        return value;
}

void sta_set_sinfo(struct sta_info *sta, struct station_info *sinfo,
                   bool tidstats)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        u32 thr = 0;
        int i, ac, cpu;
        struct ieee80211_sta_rx_stats *last_rxstats;

        last_rxstats = sta_get_last_rx_stats(sta);

        sinfo->generation = sdata->local->sta_generation;

        /* do before driver, so beacon filtering drivers have a
         * chance to e.g. just add the number of filtered beacons
         * (or just modify the value entirely, of course)
         */
        if (sdata->vif.type == NL80211_IFTYPE_STATION)
                sinfo->rx_beacon = sdata->deflink.u.mgd.count_beacon_signal;

        drv_sta_statistics(local, sdata, &sta->sta, sinfo);
        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_INACTIVE_TIME) |
                         BIT_ULL(NL80211_STA_INFO_STA_FLAGS) |
                         BIT_ULL(NL80211_STA_INFO_BSS_PARAM) |
                         BIT_ULL(NL80211_STA_INFO_CONNECTED_TIME) |
                         BIT_ULL(NL80211_STA_INFO_ASSOC_AT_BOOTTIME) |
                         BIT_ULL(NL80211_STA_INFO_RX_DROP_MISC);

        if (sdata->vif.type == NL80211_IFTYPE_STATION) {
                sinfo->beacon_loss_count =
                        sdata->deflink.u.mgd.beacon_loss_count;
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_BEACON_LOSS);
        }

        sinfo->connected_time = ktime_get_seconds() - sta->last_connected;
        sinfo->assoc_at = sta->assoc_at;
        sinfo->inactive_time =
                jiffies_to_msecs(jiffies - ieee80211_sta_last_active(sta));

        if (!(sinfo->filled & (BIT_ULL(NL80211_STA_INFO_TX_BYTES64) |
                               BIT_ULL(NL80211_STA_INFO_TX_BYTES)))) {
                sinfo->tx_bytes = 0;
                for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
                        sinfo->tx_bytes += sta->deflink.tx_stats.bytes[ac];
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES64);
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_PACKETS))) {
                sinfo->tx_packets = 0;
                for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
                        sinfo->tx_packets += sta->deflink.tx_stats.packets[ac];
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_PACKETS);
        }

        if (!(sinfo->filled & (BIT_ULL(NL80211_STA_INFO_RX_BYTES64) |
                               BIT_ULL(NL80211_STA_INFO_RX_BYTES)))) {
                sinfo->rx_bytes += sta_get_stats_bytes(&sta->deflink.rx_stats);

                if (sta->deflink.pcpu_rx_stats) {
                        for_each_possible_cpu(cpu) {
                                struct ieee80211_sta_rx_stats *cpurxs;

                                cpurxs = per_cpu_ptr(sta->deflink.pcpu_rx_stats,
                                                     cpu);
                                sinfo->rx_bytes += sta_get_stats_bytes(cpurxs);
                        }
                }

                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES64);
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_RX_PACKETS))) {
                sinfo->rx_packets = sta->deflink.rx_stats.packets;
                if (sta->deflink.pcpu_rx_stats) {
                        for_each_possible_cpu(cpu) {
                                struct ieee80211_sta_rx_stats *cpurxs;

                                cpurxs = per_cpu_ptr(sta->deflink.pcpu_rx_stats,
                                                     cpu);
                                sinfo->rx_packets += cpurxs->packets;
                        }
                }
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS);
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_RETRIES))) {
                sinfo->tx_retries = sta->deflink.status_stats.retry_count;
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES);
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_FAILED))) {
                sinfo->tx_failed = sta->deflink.status_stats.retry_failed;
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_RX_DURATION))) {
                for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
                        sinfo->rx_duration += sta->airtime[ac].rx_airtime;
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_DURATION);
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_DURATION))) {
                for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
                        sinfo->tx_duration += sta->airtime[ac].tx_airtime;
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_DURATION);
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_AIRTIME_WEIGHT))) {
                sinfo->airtime_weight = sta->airtime_weight;
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_AIRTIME_WEIGHT);
        }

        sinfo->rx_dropped_misc = sta->deflink.rx_stats.dropped;
        if (sta->deflink.pcpu_rx_stats) {
                for_each_possible_cpu(cpu) {
                        struct ieee80211_sta_rx_stats *cpurxs;

                        cpurxs = per_cpu_ptr(sta->deflink.pcpu_rx_stats, cpu);
                        sinfo->rx_dropped_misc += cpurxs->dropped;
                }
        }

        if (sdata->vif.type == NL80211_IFTYPE_STATION &&
            !(sdata->vif.driver_flags & IEEE80211_VIF_BEACON_FILTER)) {
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_BEACON_RX) |
                                 BIT_ULL(NL80211_STA_INFO_BEACON_SIGNAL_AVG);
                sinfo->rx_beacon_signal_avg = ieee80211_ave_rssi(&sdata->vif);
        }

        if (ieee80211_hw_check(&sta->local->hw, SIGNAL_DBM) ||
            ieee80211_hw_check(&sta->local->hw, SIGNAL_UNSPEC)) {
                if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_SIGNAL))) {
                        sinfo->signal = (s8)last_rxstats->last_signal;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL);
                }

                if (!sta->deflink.pcpu_rx_stats &&
                    !(sinfo->filled & BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG))) {
                        sinfo->signal_avg =
                                -ewma_signal_read(&sta->deflink.rx_stats_avg.signal);
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG);
                }
        }

        /* for the average - if pcpu_rx_stats isn't set - rxstats must point to
         * the sta->rx_stats struct, so the check here is fine with and without
         * pcpu statistics
         */
        if (last_rxstats->chains &&
            !(sinfo->filled & (BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL) |
                               BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL_AVG)))) {
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL);
                if (!sta->deflink.pcpu_rx_stats)
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL_AVG);

                sinfo->chains = last_rxstats->chains;

                for (i = 0; i < ARRAY_SIZE(sinfo->chain_signal); i++) {
                        sinfo->chain_signal[i] =
                                last_rxstats->chain_signal_last[i];
                        sinfo->chain_signal_avg[i] =
                                -ewma_signal_read(&sta->deflink.rx_stats_avg.chain_signal[i]);
                }
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_BITRATE)) &&
            !sta->sta.valid_links &&
            ieee80211_rate_valid(&sta->deflink.tx_stats.last_rate)) {
                sta_set_rate_info_tx(sta, &sta->deflink.tx_stats.last_rate,
                                     &sinfo->txrate);
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_RX_BITRATE)) &&
            !sta->sta.valid_links) {
                if (sta_set_rate_info_rx(sta, &sinfo->rxrate) == 0)
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BITRATE);
        }

        if (tidstats && !cfg80211_sinfo_alloc_tid_stats(sinfo, GFP_KERNEL)) {
                for (i = 0; i < IEEE80211_NUM_TIDS + 1; i++)
                        sta_set_tidstats(sta, &sinfo->pertid[i], i);
        }

        if (ieee80211_vif_is_mesh(&sdata->vif)) {
#ifdef CONFIG_MAC80211_MESH
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_LLID) |
                                 BIT_ULL(NL80211_STA_INFO_PLID) |
                                 BIT_ULL(NL80211_STA_INFO_PLINK_STATE) |
                                 BIT_ULL(NL80211_STA_INFO_LOCAL_PM) |
                                 BIT_ULL(NL80211_STA_INFO_PEER_PM) |
                                 BIT_ULL(NL80211_STA_INFO_NONPEER_PM) |
                                 BIT_ULL(NL80211_STA_INFO_CONNECTED_TO_GATE) |
                                 BIT_ULL(NL80211_STA_INFO_CONNECTED_TO_AS);

                sinfo->llid = sta->mesh->llid;
                sinfo->plid = sta->mesh->plid;
                sinfo->plink_state = sta->mesh->plink_state;
                if (test_sta_flag(sta, WLAN_STA_TOFFSET_KNOWN)) {
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_T_OFFSET);
                        sinfo->t_offset = sta->mesh->t_offset;
                }
                sinfo->local_pm = sta->mesh->local_pm;
                sinfo->peer_pm = sta->mesh->peer_pm;
                sinfo->nonpeer_pm = sta->mesh->nonpeer_pm;
                sinfo->connected_to_gate = sta->mesh->connected_to_gate;
                sinfo->connected_to_as = sta->mesh->connected_to_as;
#endif
        }

        sinfo->bss_param.flags = 0;
        if (sdata->vif.bss_conf.use_cts_prot)
                sinfo->bss_param.flags |= BSS_PARAM_FLAGS_CTS_PROT;
        if (sdata->vif.bss_conf.use_short_preamble)
                sinfo->bss_param.flags |= BSS_PARAM_FLAGS_SHORT_PREAMBLE;
        if (sdata->vif.bss_conf.use_short_slot)
                sinfo->bss_param.flags |= BSS_PARAM_FLAGS_SHORT_SLOT_TIME;
        sinfo->bss_param.dtim_period = sdata->vif.bss_conf.dtim_period;
        sinfo->bss_param.beacon_interval = sdata->vif.bss_conf.beacon_int;

        sinfo->sta_flags.set = 0;
        sinfo->sta_flags.mask = BIT(NL80211_STA_FLAG_AUTHORIZED) |
                                BIT(NL80211_STA_FLAG_SHORT_PREAMBLE) |
                                BIT(NL80211_STA_FLAG_WME) |
                                BIT(NL80211_STA_FLAG_MFP) |
                                BIT(NL80211_STA_FLAG_AUTHENTICATED) |
                                BIT(NL80211_STA_FLAG_ASSOCIATED) |
                                BIT(NL80211_STA_FLAG_TDLS_PEER);
        if (test_sta_flag(sta, WLAN_STA_AUTHORIZED))
                sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_AUTHORIZED);
        if (test_sta_flag(sta, WLAN_STA_SHORT_PREAMBLE))
                sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_SHORT_PREAMBLE);
        if (sta->sta.wme)
                sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_WME);
        if (test_sta_flag(sta, WLAN_STA_MFP))
                sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_MFP);
        if (test_sta_flag(sta, WLAN_STA_AUTH))
                sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_AUTHENTICATED);
        if (test_sta_flag(sta, WLAN_STA_ASSOC))
                sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_ASSOCIATED);
        if (test_sta_flag(sta, WLAN_STA_TDLS_PEER))
                sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_TDLS_PEER);

        thr = sta_get_expected_throughput(sta);

        if (thr != 0) {
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_EXPECTED_THROUGHPUT);
                sinfo->expected_throughput = thr;
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL)) &&
            sta->deflink.status_stats.ack_signal_filled) {
                sinfo->ack_signal = sta->deflink.status_stats.last_ack_signal;
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL);
        }

        if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL_AVG)) &&
            sta->deflink.status_stats.ack_signal_filled) {
                sinfo->avg_ack_signal =
                        -(s8)ewma_avg_signal_read(
                                &sta->deflink.status_stats.avg_ack_signal);
                sinfo->filled |=
                        BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL_AVG);
        }

        if (ieee80211_vif_is_mesh(&sdata->vif)) {
                sinfo->filled |= BIT_ULL(NL80211_STA_INFO_AIRTIME_LINK_METRIC);
                sinfo->airtime_link_metric =
                        airtime_link_metric_get(local, sta);
        }
}

u32 sta_get_expected_throughput(struct sta_info *sta)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        struct rate_control_ref *ref = NULL;
        u32 thr = 0;

        if (test_sta_flag(sta, WLAN_STA_RATE_CONTROL))
                ref = local->rate_ctrl;

        /* check if the driver has a SW RC implementation */
        if (ref && ref->ops->get_expected_throughput)
                thr = ref->ops->get_expected_throughput(sta->rate_ctrl_priv);
        else
                thr = drv_get_expected_throughput(local, sta);

        return thr;
}

unsigned long ieee80211_sta_last_active(struct sta_info *sta)
{
        struct ieee80211_sta_rx_stats *stats = sta_get_last_rx_stats(sta);

        if (!sta->deflink.status_stats.last_ack ||
            time_after(stats->last_rx, sta->deflink.status_stats.last_ack))
                return stats->last_rx;
        return sta->deflink.status_stats.last_ack;
}

static void sta_update_codel_params(struct sta_info *sta, u32 thr)
{
        if (thr && thr < STA_SLOW_THRESHOLD * sta->local->num_sta) {
                sta->cparams.target = MS2TIME(50);
                sta->cparams.interval = MS2TIME(300);
                sta->cparams.ecn = false;
        } else {
                sta->cparams.target = MS2TIME(20);
                sta->cparams.interval = MS2TIME(100);
                sta->cparams.ecn = true;
        }
}

void ieee80211_sta_set_expected_throughput(struct ieee80211_sta *pubsta,
                                           u32 thr)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);

        sta_update_codel_params(sta, thr);
}

int ieee80211_sta_allocate_link(struct sta_info *sta, unsigned int link_id)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct sta_link_alloc *alloc;
        int ret;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        WARN_ON(!test_sta_flag(sta, WLAN_STA_INSERTED));

        /* must represent an MLD from the start */
        if (WARN_ON(!sta->sta.valid_links))
                return -EINVAL;

        if (WARN_ON(sta->sta.valid_links & BIT(link_id) ||
                    sta->link[link_id]))
                return -EBUSY;

        alloc = kzalloc(sizeof(*alloc), GFP_KERNEL);
        if (!alloc)
                return -ENOMEM;

        ret = sta_info_alloc_link(sdata->local, &alloc->info, GFP_KERNEL);
        if (ret) {
                kfree(alloc);
                return ret;
        }

        sta_info_add_link(sta, link_id, &alloc->info, &alloc->sta);

        ieee80211_link_sta_debugfs_add(&alloc->info);

        return 0;
}

void ieee80211_sta_free_link(struct sta_info *sta, unsigned int link_id)
{
        lockdep_assert_wiphy(sta->sdata->local->hw.wiphy);

        WARN_ON(!test_sta_flag(sta, WLAN_STA_INSERTED));

        sta_remove_link(sta, link_id, false);
}

int ieee80211_sta_activate_link(struct sta_info *sta, unsigned int link_id)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct link_sta_info *link_sta;
        u16 old_links = sta->sta.valid_links;
        u16 new_links = old_links | BIT(link_id);
        int ret;

        link_sta = rcu_dereference_protected(sta->link[link_id],
                                             lockdep_is_held(&sdata->local->hw.wiphy->mtx));

        if (WARN_ON(old_links == new_links || !link_sta))
                return -EINVAL;

        rcu_read_lock();
        if (link_sta_info_hash_lookup(sdata->local, link_sta->addr)) {
                rcu_read_unlock();
                return -EALREADY;
        }
        /* we only modify under the mutex so this is fine */
        rcu_read_unlock();

        sta->sta.valid_links = new_links;

        if (WARN_ON(!test_sta_flag(sta, WLAN_STA_INSERTED)))
                goto hash;

        ieee80211_recalc_min_chandef(sdata, link_id);

        /* Ensure the values are updated for the driver,
         * redone by sta_remove_link on failure.
         */
        ieee80211_sta_recalc_aggregates(&sta->sta);

        ret = drv_change_sta_links(sdata->local, sdata, &sta->sta,
                                   old_links, new_links);
        if (ret) {
                sta->sta.valid_links = old_links;
                sta_remove_link(sta, link_id, false);
                return ret;
        }

hash:
        ret = link_sta_info_hash_add(sdata->local, link_sta);
        WARN_ON(ret);
        return 0;
}

void ieee80211_sta_remove_link(struct sta_info *sta, unsigned int link_id)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        u16 old_links = sta->sta.valid_links;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        sta->sta.valid_links &= ~BIT(link_id);

        if (!WARN_ON(!test_sta_flag(sta, WLAN_STA_INSERTED)))
                drv_change_sta_links(sdata->local, sdata, &sta->sta,
                                     old_links, sta->sta.valid_links);

        sta_remove_link(sta, link_id, true);
}

void ieee80211_sta_set_max_amsdu_subframes(struct sta_info *sta,
                                           const u8 *ext_capab,
                                           unsigned int ext_capab_len)
{
        u8 val;

        sta->sta.max_amsdu_subframes = 0;

        if (ext_capab_len < 8)
                return;

        /* The sender might not have sent the last bit, consider it to be 0 */
        val = u8_get_bits(ext_capab[7], WLAN_EXT_CAPA8_MAX_MSDU_IN_AMSDU_LSB);

        /* we did get all the bits, take the MSB as well */
        if (ext_capab_len >= 9)
                val |= u8_get_bits(ext_capab[8],
                                   WLAN_EXT_CAPA9_MAX_MSDU_IN_AMSDU_MSB) << 1;

        if (val)
                sta->sta.max_amsdu_subframes = 4 << (4 - val);
}

#ifdef CONFIG_LOCKDEP
bool lockdep_sta_mutex_held(struct ieee80211_sta *pubsta)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);

        return lockdep_is_held(&sta->local->hw.wiphy->mtx);
}
EXPORT_SYMBOL(lockdep_sta_mutex_held);
#endif




























































































































































































































































































































































































































































































































































































































































































































  156 
  155 

  133 



  131 
  133 



  132 




  132 





  133 













































































   65 









   65 




















   74 

   65 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
// SPDX-License-Identifier: GPL-2.0
/* sysfs entries for device PM */
#include <linux/device.h>
#include <linux/kobject.h>
#include <linux/string.h>
#include <linux/export.h>
#include <linux/pm_qos.h>
#include <linux/pm_runtime.h>
#include <linux/pm_wakeup.h>
#include <linux/atomic.h>
#include <linux/jiffies.h>
#include "power.h"

/*
 *        control - Report/change current runtime PM setting of the device
 *
 *        Runtime power management of a device can be blocked with the help of
 *        this attribute.  All devices have one of the following two values for
 *        the power/control file:
 *
 *         + "auto\n" to allow the device to be power managed at run time;
 *         + "on\n" to prevent the device from being power managed at run time;
 *
 *        The default for all devices is "auto", which means that devices may be
 *        subject to automatic power management, depending on their drivers.
 *        Changing this attribute to "on" prevents the driver from power managing
 *        the device at run time.  Doing that while the device is suspended causes
 *        it to be woken up.
 *
 *        wakeup - Report/change current wakeup option for device
 *
 *        Some devices support "wakeup" events, which are hardware signals
 *        used to activate devices from suspended or low power states.  Such
 *        devices have one of three values for the sysfs power/wakeup file:
 *
 *         + "enabled\n" to issue the events;
 *         + "disabled\n" not to do so; or
 *         + "\n" for temporary or permanent inability to issue wakeup.
 *
 *        (For example, unconfigured USB devices can't issue wakeups.)
 *
 *        Familiar examples of devices that can issue wakeup events include
 *        keyboards and mice (both PS2 and USB styles), power buttons, modems,
 *        "Wake-On-LAN" Ethernet links, GPIO lines, and more.  Some events
 *        will wake the entire system from a suspend state; others may just
 *        wake up the device (if the system as a whole is already active).
 *        Some wakeup events use normal IRQ lines; other use special out
 *        of band signaling.
 *
 *        It is the responsibility of device drivers to enable (or disable)
 *        wakeup signaling as part of changing device power states, respecting
 *        the policy choices provided through the driver model.
 *
 *        Devices may not be able to generate wakeup events from all power
 *        states.  Also, the events may be ignored in some configurations;
 *        for example, they might need help from other devices that aren't
 *        active, or which may have wakeup disabled.  Some drivers rely on
 *        wakeup events internally (unless they are disabled), keeping
 *        their hardware in low power modes whenever they're unused.  This
 *        saves runtime power, without requiring system-wide sleep states.
 *
 *        async - Report/change current async suspend setting for the device
 *
 *        Asynchronous suspend and resume of the device during system-wide power
 *        state transitions can be enabled by writing "enabled" to this file.
 *        Analogously, if "disabled" is written to this file, the device will be
 *        suspended and resumed synchronously.
 *
 *        All devices have one of the following two values for power/async:
 *
 *         + "enabled\n" to permit the asynchronous suspend/resume of the device;
 *         + "disabled\n" to forbid it;
 *
 *        NOTE: It generally is unsafe to permit the asynchronous suspend/resume
 *        of a device unless it is certain that all of the PM dependencies of the
 *        device are known to the PM core.  However, for some devices this
 *        attribute is set to "enabled" by bus type code or device drivers and in
 *        that cases it should be safe to leave the default value.
 *
 *        autosuspend_delay_ms - Report/change a device's autosuspend_delay value
 *
 *        Some drivers don't want to carry out a runtime suspend as soon as a
 *        device becomes idle; they want it always to remain idle for some period
 *        of time before suspending it.  This period is the autosuspend_delay
 *        value (expressed in milliseconds) and it can be controlled by the user.
 *        If the value is negative then the device will never be runtime
 *        suspended.
 *
 *        NOTE: The autosuspend_delay_ms attribute and the autosuspend_delay
 *        value are used only if the driver calls pm_runtime_use_autosuspend().
 *
 *        wakeup_count - Report the number of wakeup events related to the device
 */

const char power_group_name[] = "power";
EXPORT_SYMBOL_GPL(power_group_name);

static const char ctrl_auto[] = "auto";
static const char ctrl_on[] = "on";

static ssize_t control_show(struct device *dev, struct device_attribute *attr,
                            char *buf)
{
        return sysfs_emit(buf, "%s\n",
                          dev->power.runtime_auto ? ctrl_auto : ctrl_on);
}

static ssize_t control_store(struct device * dev, struct device_attribute *attr,
                             const char * buf, size_t n)
{
        device_lock(dev);
        if (sysfs_streq(buf, ctrl_auto))
                pm_runtime_allow(dev);
        else if (sysfs_streq(buf, ctrl_on))
                pm_runtime_forbid(dev);
        else
                n = -EINVAL;
        device_unlock(dev);
        return n;
}

static DEVICE_ATTR_RW(control);

static ssize_t runtime_active_time_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        u64 tmp = pm_runtime_active_time(dev);

        do_div(tmp, NSEC_PER_MSEC);

        return sysfs_emit(buf, "%llu\n", tmp);
}

static DEVICE_ATTR_RO(runtime_active_time);

static ssize_t runtime_suspended_time_show(struct device *dev,
                                           struct device_attribute *attr,
                                           char *buf)
{
        u64 tmp = pm_runtime_suspended_time(dev);

        do_div(tmp, NSEC_PER_MSEC);

        return sysfs_emit(buf, "%llu\n", tmp);
}

static DEVICE_ATTR_RO(runtime_suspended_time);

static ssize_t runtime_status_show(struct device *dev,
                                   struct device_attribute *attr, char *buf)
{
        const char *output;

        if (dev->power.runtime_error) {
                output = "error";
        } else if (dev->power.disable_depth) {
                output = "unsupported";
        } else {
                switch (dev->power.runtime_status) {
                case RPM_SUSPENDED:
                        output = "suspended";
                        break;
                case RPM_SUSPENDING:
                        output = "suspending";
                        break;
                case RPM_RESUMING:
                        output = "resuming";
                        break;
                case RPM_ACTIVE:
                        output = "active";
                        break;
                default:
                        return -EIO;
                }
        }
        return sysfs_emit(buf, "%s\n", output);
}

static DEVICE_ATTR_RO(runtime_status);

static ssize_t autosuspend_delay_ms_show(struct device *dev,
                                         struct device_attribute *attr,
                                         char *buf)
{
        if (!dev->power.use_autosuspend)
                return -EIO;

        return sysfs_emit(buf, "%d\n", dev->power.autosuspend_delay);
}

static ssize_t autosuspend_delay_ms_store(struct device *dev,
                struct device_attribute *attr, const char *buf, size_t n)
{
        long delay;

        if (!dev->power.use_autosuspend)
                return -EIO;

        if (kstrtol(buf, 10, &delay) != 0 || delay != (int) delay)
                return -EINVAL;

        device_lock(dev);
        pm_runtime_set_autosuspend_delay(dev, delay);
        device_unlock(dev);
        return n;
}

static DEVICE_ATTR_RW(autosuspend_delay_ms);

static ssize_t pm_qos_resume_latency_us_show(struct device *dev,
                                             struct device_attribute *attr,
                                             char *buf)
{
        s32 value = dev_pm_qos_requested_resume_latency(dev);

        if (value == 0)
                return sysfs_emit(buf, "n/a\n");
        if (value == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT)
                value = 0;

        return sysfs_emit(buf, "%d\n", value);
}

static ssize_t pm_qos_resume_latency_us_store(struct device *dev,
                                              struct device_attribute *attr,
                                              const char *buf, size_t n)
{
        s32 value;
        int ret;

        if (!kstrtos32(buf, 0, &value)) {
                /*
                 * Prevent users from writing negative or "no constraint" values
                 * directly.
                 */
                if (value < 0 || value == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT)
                        return -EINVAL;

                if (value == 0)
                        value = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT;
        } else if (sysfs_streq(buf, "n/a")) {
                value = 0;
        } else {
                return -EINVAL;
        }

        ret = dev_pm_qos_update_request(dev->power.qos->resume_latency_req,
                                        value);
        return ret < 0 ? ret : n;
}

static DEVICE_ATTR_RW(pm_qos_resume_latency_us);

static ssize_t pm_qos_latency_tolerance_us_show(struct device *dev,
                                                struct device_attribute *attr,
                                                char *buf)
{
        s32 value = dev_pm_qos_get_user_latency_tolerance(dev);

        if (value < 0)
                return sysfs_emit(buf, "%s\n", "auto");
        if (value == PM_QOS_LATENCY_ANY)
                return sysfs_emit(buf, "%s\n", "any");

        return sysfs_emit(buf, "%d\n", value);
}

static ssize_t pm_qos_latency_tolerance_us_store(struct device *dev,
                                                 struct device_attribute *attr,
                                                 const char *buf, size_t n)
{
        s32 value;
        int ret;

        if (kstrtos32(buf, 0, &value) == 0) {
                /* Users can't write negative values directly */
                if (value < 0)
                        return -EINVAL;
        } else {
                if (sysfs_streq(buf, "auto"))
                        value = PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT;
                else if (sysfs_streq(buf, "any"))
                        value = PM_QOS_LATENCY_ANY;
                else
                        return -EINVAL;
        }
        ret = dev_pm_qos_update_user_latency_tolerance(dev, value);
        return ret < 0 ? ret : n;
}

static DEVICE_ATTR_RW(pm_qos_latency_tolerance_us);

static ssize_t pm_qos_no_power_off_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        return sysfs_emit(buf, "%d\n", !!(dev_pm_qos_requested_flags(dev)
                                          & PM_QOS_FLAG_NO_POWER_OFF));
}

static ssize_t pm_qos_no_power_off_store(struct device *dev,
                                         struct device_attribute *attr,
                                         const char *buf, size_t n)
{
        int ret;

        if (kstrtoint(buf, 0, &ret))
                return -EINVAL;

        if (ret != 0 && ret != 1)
                return -EINVAL;

        ret = dev_pm_qos_update_flags(dev, PM_QOS_FLAG_NO_POWER_OFF, ret);
        return ret < 0 ? ret : n;
}

static DEVICE_ATTR_RW(pm_qos_no_power_off);

#ifdef CONFIG_PM_SLEEP
static const char _enabled[] = "enabled";
static const char _disabled[] = "disabled";

static ssize_t wakeup_show(struct device *dev, struct device_attribute *attr,
                           char *buf)
{
        return sysfs_emit(buf, "%s\n", device_can_wakeup(dev)
                          ? (device_may_wakeup(dev) ? _enabled : _disabled)
                          : "");
}

static ssize_t wakeup_store(struct device *dev, struct device_attribute *attr,
                            const char *buf, size_t n)
{
        if (!device_can_wakeup(dev))
                return -EINVAL;

        if (sysfs_streq(buf, _enabled))
                device_set_wakeup_enable(dev, 1);
        else if (sysfs_streq(buf, _disabled))
                device_set_wakeup_enable(dev, 0);
        else
                return -EINVAL;
        return n;
}

static DEVICE_ATTR_RW(wakeup);

static ssize_t wakeup_count_show(struct device *dev,
                                 struct device_attribute *attr, char *buf)
{
        unsigned long count;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                count = dev->power.wakeup->wakeup_count;
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lu\n", count);
}

static DEVICE_ATTR_RO(wakeup_count);

static ssize_t wakeup_active_count_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        unsigned long count;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                count = dev->power.wakeup->active_count;
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lu\n", count);
}

static DEVICE_ATTR_RO(wakeup_active_count);

static ssize_t wakeup_abort_count_show(struct device *dev,
                                       struct device_attribute *attr,
                                       char *buf)
{
        unsigned long count;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                count = dev->power.wakeup->wakeup_count;
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lu\n", count);
}

static DEVICE_ATTR_RO(wakeup_abort_count);

static ssize_t wakeup_expire_count_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        unsigned long count;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                count = dev->power.wakeup->expire_count;
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lu\n", count);
}

static DEVICE_ATTR_RO(wakeup_expire_count);

static ssize_t wakeup_active_show(struct device *dev,
                                  struct device_attribute *attr, char *buf)
{
        unsigned int active;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                active = dev->power.wakeup->active;
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%u\n", active);
}

static DEVICE_ATTR_RO(wakeup_active);

static ssize_t wakeup_total_time_ms_show(struct device *dev,
                                         struct device_attribute *attr,
                                         char *buf)
{
        s64 msec;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                msec = ktime_to_ms(dev->power.wakeup->total_time);
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lld\n", msec);
}

static DEVICE_ATTR_RO(wakeup_total_time_ms);

static ssize_t wakeup_max_time_ms_show(struct device *dev,
                                       struct device_attribute *attr, char *buf)
{
        s64 msec;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                msec = ktime_to_ms(dev->power.wakeup->max_time);
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lld\n", msec);
}

static DEVICE_ATTR_RO(wakeup_max_time_ms);

static ssize_t wakeup_last_time_ms_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        s64 msec;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                msec = ktime_to_ms(dev->power.wakeup->last_time);
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lld\n", msec);
}

static DEVICE_ATTR_RO(wakeup_last_time_ms);

#ifdef CONFIG_PM_AUTOSLEEP
static ssize_t wakeup_prevent_sleep_time_ms_show(struct device *dev,
                                                 struct device_attribute *attr,
                                                 char *buf)
{
        s64 msec;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                msec = ktime_to_ms(dev->power.wakeup->prevent_sleep_time);
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lld\n", msec);
}

static DEVICE_ATTR_RO(wakeup_prevent_sleep_time_ms);
#endif /* CONFIG_PM_AUTOSLEEP */

static inline int dpm_sysfs_wakeup_change_owner(struct device *dev, kuid_t kuid,
                                                kgid_t kgid)
{
        if (dev->power.wakeup && dev->power.wakeup->dev)
                return device_change_owner(dev->power.wakeup->dev, kuid, kgid);
        return 0;
}

#else /* CONFIG_PM_SLEEP */
static inline int dpm_sysfs_wakeup_change_owner(struct device *dev, kuid_t kuid,
                                                kgid_t kgid)
{
        return 0;
}
#endif

#ifdef CONFIG_PM_ADVANCED_DEBUG
static ssize_t runtime_usage_show(struct device *dev,
                                  struct device_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%d\n", atomic_read(&dev->power.usage_count));
}
static DEVICE_ATTR_RO(runtime_usage);

static ssize_t runtime_active_kids_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        return sysfs_emit(buf, "%d\n", dev->power.ignore_children ?
                          0 : atomic_read(&dev->power.child_count));
}
static DEVICE_ATTR_RO(runtime_active_kids);

static ssize_t runtime_enabled_show(struct device *dev,
                                    struct device_attribute *attr, char *buf)
{
        const char *output;

        if (dev->power.disable_depth && !dev->power.runtime_auto)
                output = "disabled & forbidden";
        else if (dev->power.disable_depth)
                output = "disabled";
        else if (!dev->power.runtime_auto)
                output = "forbidden";
        else
                output = "enabled";

        return sysfs_emit(buf, "%s\n", output);
}
static DEVICE_ATTR_RO(runtime_enabled);

#ifdef CONFIG_PM_SLEEP
static ssize_t async_show(struct device *dev, struct device_attribute *attr,
                          char *buf)
{
        return sysfs_emit(buf, "%s\n",
                          device_async_suspend_enabled(dev) ?
                          _enabled : _disabled);
}

static ssize_t async_store(struct device *dev, struct device_attribute *attr,
                           const char *buf, size_t n)
{
        if (sysfs_streq(buf, _enabled))
                device_enable_async_suspend(dev);
        else if (sysfs_streq(buf, _disabled))
                device_disable_async_suspend(dev);
        else
                return -EINVAL;
        return n;
}

static DEVICE_ATTR_RW(async);

#endif /* CONFIG_PM_SLEEP */
#endif /* CONFIG_PM_ADVANCED_DEBUG */

static struct attribute *power_attrs[] = {
#ifdef CONFIG_PM_ADVANCED_DEBUG
#ifdef CONFIG_PM_SLEEP
        &dev_attr_async.attr,
#endif
        &dev_attr_runtime_status.attr,
        &dev_attr_runtime_usage.attr,
        &dev_attr_runtime_active_kids.attr,
        &dev_attr_runtime_enabled.attr,
#endif /* CONFIG_PM_ADVANCED_DEBUG */
        NULL,
};
static const struct attribute_group pm_attr_group = {
        .name        = power_group_name,
        .attrs        = power_attrs,
};

static struct attribute *wakeup_attrs[] = {
#ifdef CONFIG_PM_SLEEP
        &dev_attr_wakeup.attr,
        &dev_attr_wakeup_count.attr,
        &dev_attr_wakeup_active_count.attr,
        &dev_attr_wakeup_abort_count.attr,
        &dev_attr_wakeup_expire_count.attr,
        &dev_attr_wakeup_active.attr,
        &dev_attr_wakeup_total_time_ms.attr,
        &dev_attr_wakeup_max_time_ms.attr,
        &dev_attr_wakeup_last_time_ms.attr,
#ifdef CONFIG_PM_AUTOSLEEP
        &dev_attr_wakeup_prevent_sleep_time_ms.attr,
#endif
#endif
        NULL,
};
static const struct attribute_group pm_wakeup_attr_group = {
        .name        = power_group_name,
        .attrs        = wakeup_attrs,
};

static struct attribute *runtime_attrs[] = {
#ifndef CONFIG_PM_ADVANCED_DEBUG
        &dev_attr_runtime_status.attr,
#endif
        &dev_attr_control.attr,
        &dev_attr_runtime_suspended_time.attr,
        &dev_attr_runtime_active_time.attr,
        &dev_attr_autosuspend_delay_ms.attr,
        NULL,
};
static const struct attribute_group pm_runtime_attr_group = {
        .name        = power_group_name,
        .attrs        = runtime_attrs,
};

static struct attribute *pm_qos_resume_latency_attrs[] = {
        &dev_attr_pm_qos_resume_latency_us.attr,
        NULL,
};
static const struct attribute_group pm_qos_resume_latency_attr_group = {
        .name        = power_group_name,
        .attrs        = pm_qos_resume_latency_attrs,
};

static struct attribute *pm_qos_latency_tolerance_attrs[] = {
        &dev_attr_pm_qos_latency_tolerance_us.attr,
        NULL,
};
static const struct attribute_group pm_qos_latency_tolerance_attr_group = {
        .name        = power_group_name,
        .attrs        = pm_qos_latency_tolerance_attrs,
};

static struct attribute *pm_qos_flags_attrs[] = {
        &dev_attr_pm_qos_no_power_off.attr,
        NULL,
};
static const struct attribute_group pm_qos_flags_attr_group = {
        .name        = power_group_name,
        .attrs        = pm_qos_flags_attrs,
};

int dpm_sysfs_add(struct device *dev)
{
        int rc;

        /* No need to create PM sysfs if explicitly disabled. */
        if (device_pm_not_required(dev))
                return 0;

        rc = sysfs_create_group(&dev->kobj, &pm_attr_group);
        if (rc)
                return rc;

        if (!pm_runtime_has_no_callbacks(dev)) {
                rc = sysfs_merge_group(&dev->kobj, &pm_runtime_attr_group);
                if (rc)
                        goto err_out;
        }
        if (device_can_wakeup(dev)) {
                rc = sysfs_merge_group(&dev->kobj, &pm_wakeup_attr_group);
                if (rc)
                        goto err_runtime;
        }
        if (dev->power.set_latency_tolerance) {
                rc = sysfs_merge_group(&dev->kobj,
                                       &pm_qos_latency_tolerance_attr_group);
                if (rc)
                        goto err_wakeup;
        }
        rc = pm_wakeup_source_sysfs_add(dev);
        if (rc)
                goto err_latency;
        return 0;

 err_latency:
        sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group);
 err_wakeup:
        sysfs_unmerge_group(&dev->kobj, &pm_wakeup_attr_group);
 err_runtime:
        sysfs_unmerge_group(&dev->kobj, &pm_runtime_attr_group);
 err_out:
        sysfs_remove_group(&dev->kobj, &pm_attr_group);
        return rc;
}

int dpm_sysfs_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid)
{
        int rc;

        if (device_pm_not_required(dev))
                return 0;

        rc = sysfs_group_change_owner(&dev->kobj, &pm_attr_group, kuid, kgid);
        if (rc)
                return rc;

        if (!pm_runtime_has_no_callbacks(dev)) {
                rc = sysfs_group_change_owner(
                        &dev->kobj, &pm_runtime_attr_group, kuid, kgid);
                if (rc)
                        return rc;
        }

        if (device_can_wakeup(dev)) {
                rc = sysfs_group_change_owner(&dev->kobj, &pm_wakeup_attr_group,
                                              kuid, kgid);
                if (rc)
                        return rc;

                rc = dpm_sysfs_wakeup_change_owner(dev, kuid, kgid);
                if (rc)
                        return rc;
        }

        if (dev->power.set_latency_tolerance) {
                rc = sysfs_group_change_owner(
                        &dev->kobj, &pm_qos_latency_tolerance_attr_group, kuid,
                        kgid);
                if (rc)
                        return rc;
        }
        return 0;
}

int wakeup_sysfs_add(struct device *dev)
{
        int ret = sysfs_merge_group(&dev->kobj, &pm_wakeup_attr_group);

        if (!ret)
                kobject_uevent(&dev->kobj, KOBJ_CHANGE);

        return ret;
}

void wakeup_sysfs_remove(struct device *dev)
{
        sysfs_unmerge_group(&dev->kobj, &pm_wakeup_attr_group);
        kobject_uevent(&dev->kobj, KOBJ_CHANGE);
}

int pm_qos_sysfs_add_resume_latency(struct device *dev)
{
        return sysfs_merge_group(&dev->kobj, &pm_qos_resume_latency_attr_group);
}

void pm_qos_sysfs_remove_resume_latency(struct device *dev)
{
        sysfs_unmerge_group(&dev->kobj, &pm_qos_resume_latency_attr_group);
}

int pm_qos_sysfs_add_flags(struct device *dev)
{
        return sysfs_merge_group(&dev->kobj, &pm_qos_flags_attr_group);
}

void pm_qos_sysfs_remove_flags(struct device *dev)
{
        sysfs_unmerge_group(&dev->kobj, &pm_qos_flags_attr_group);
}

int pm_qos_sysfs_add_latency_tolerance(struct device *dev)
{
        return sysfs_merge_group(&dev->kobj,
                                 &pm_qos_latency_tolerance_attr_group);
}

void pm_qos_sysfs_remove_latency_tolerance(struct device *dev)
{
        sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group);
}

void rpm_sysfs_remove(struct device *dev)
{
        sysfs_unmerge_group(&dev->kobj, &pm_runtime_attr_group);
}

void dpm_sysfs_remove(struct device *dev)
{
        if (device_pm_not_required(dev))
                return;
        sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group);
        dev_pm_qos_constraints_destroy(dev);
        rpm_sysfs_remove(dev);
        sysfs_unmerge_group(&dev->kobj, &pm_wakeup_attr_group);
        sysfs_remove_group(&dev->kobj, &pm_attr_group);
}

















    6 









































































































































   20 





















   31 




   31 
   23 


   23 




    8 
    8 

    8 




    4 
    4 

    4 
    1 


    4 


    8 



   31 

























   43 
   42 
   42 

   43 

   43 

   42 
   23 






   23 


    6 

   23 
   21 


   21 


    4 





   20 
   20 

   20 
   20 

   20 

   13 


   13 

   13 
   13 

   13 
    3 

   13 

   20 


   43 



    6 
















































































    3 


    3 




    1 

















    1 









    1 






    3 



























































































   15 







    4 








    4 
    4 




























    4 































   15 






    4 




    3 


    3 





































    7 






    7 
    7 



    6 




    1 
























    5 




    5 





    5 



    5 

    5 
    3 





    3 








































































































































    9 








   10 







   12 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
// SPDX-License-Identifier: GPL-2.0-only

#include <linux/ethtool_netlink.h>
#include <linux/bitmap.h>
#include "netlink.h"
#include "bitset.h"

/* Some bitmaps are internally represented as an array of unsigned long, some
 * as an array of u32 (some even as single u32 for now). To avoid the need of
 * wrappers on caller side, we provide two set of functions: those with "32"
 * suffix in their names expect u32 based bitmaps, those without it expect
 * unsigned long bitmaps.
 */

static u32 ethnl_lower_bits(unsigned int n)
{
        return ~(u32)0 >> (32 - n % 32);
}

static u32 ethnl_upper_bits(unsigned int n)
{
        return ~(u32)0 << (n % 32);
}

/**
 * ethnl_bitmap32_clear() - Clear u32 based bitmap
 * @dst:   bitmap to clear
 * @start: beginning of the interval
 * @end:   end of the interval
 * @mod:   set if bitmap was modified
 *
 * Clear @nbits bits of a bitmap with indices @start <= i < @end
 */
static void ethnl_bitmap32_clear(u32 *dst, unsigned int start, unsigned int end,
                                 bool *mod)
{
        unsigned int start_word = start / 32;
        unsigned int end_word = end / 32;
        unsigned int i;
        u32 mask;

        if (end <= start)
                return;

        if (start % 32) {
                mask = ethnl_upper_bits(start);
                if (end_word == start_word) {
                        mask &= ethnl_lower_bits(end);
                        if (dst[start_word] & mask) {
                                dst[start_word] &= ~mask;
                                *mod = true;
                        }
                        return;
                }
                if (dst[start_word] & mask) {
                        dst[start_word] &= ~mask;
                        *mod = true;
                }
                start_word++;
        }

        for (i = start_word; i < end_word; i++) {
                if (dst[i]) {
                        dst[i] = 0;
                        *mod = true;
                }
        }
        if (end % 32) {
                mask = ethnl_lower_bits(end);
                if (dst[end_word] & mask) {
                        dst[end_word] &= ~mask;
                        *mod = true;
                }
        }
}

/**
 * ethnl_bitmap32_not_zero() - Check if any bit is set in an interval
 * @map:   bitmap to test
 * @start: beginning of the interval
 * @end:   end of the interval
 *
 * Return: true if there is non-zero bit with  index @start <= i < @end,
 *         false if the whole interval is zero
 */
static bool ethnl_bitmap32_not_zero(const u32 *map, unsigned int start,
                                    unsigned int end)
{
        unsigned int start_word = start / 32;
        unsigned int end_word = end / 32;
        u32 mask;

        if (end <= start)
                return true;

        if (start % 32) {
                mask = ethnl_upper_bits(start);
                if (end_word == start_word) {
                        mask &= ethnl_lower_bits(end);
                        return map[start_word] & mask;
                }
                if (map[start_word] & mask)
                        return true;
                start_word++;
        }

        if (!memchr_inv(map + start_word, '\0',
                        (end_word - start_word) * sizeof(u32)))
                return true;
        if (end % 32 == 0)
                return true;
        return map[end_word] & ethnl_lower_bits(end);
}

/**
 * ethnl_bitmap32_update() - Modify u32 based bitmap according to value/mask
 *                             pair
 * @dst:   bitmap to update
 * @nbits: bit size of the bitmap
 * @value: values to set
 * @mask:  mask of bits to set
 * @mod:   set to true if bitmap is modified, preserve if not
 *
 * Set bits in @dst bitmap which are set in @mask to values from @value, leave
 * the rest untouched. If destination bitmap was modified, set @mod to true,
 * leave as it is if not.
 */
static void ethnl_bitmap32_update(u32 *dst, unsigned int nbits,
                                  const u32 *value, const u32 *mask, bool *mod)
{
        while (nbits > 0) {
                u32 real_mask = mask ? *mask : ~(u32)0;
                u32 new_value;

                if (nbits < 32)
                        real_mask &= ethnl_lower_bits(nbits);
                new_value = (*dst & ~real_mask) | (*value & real_mask);
                if (new_value != *dst) {
                        *dst = new_value;
                        *mod = true;
                }

                if (nbits <= 32)
                        break;
                dst++;
                nbits -= 32;
                value++;
                if (mask)
                        mask++;
        }
}

static bool ethnl_bitmap32_test_bit(const u32 *map, unsigned int index)
{
        return map[index / 32] & (1U << (index % 32));
}

/**
 * ethnl_bitset32_size() - Calculate size of bitset nested attribute
 * @val:     value bitmap (u32 based)
 * @mask:    mask bitmap (u32 based, optional)
 * @nbits:   bit length of the bitset
 * @names:   array of bit names (optional)
 * @compact: assume compact format for output
 *
 * Estimate length of netlink attribute composed by a later call to
 * ethnl_put_bitset32() call with the same arguments.
 *
 * Return: negative error code or attribute length estimate
 */
int ethnl_bitset32_size(const u32 *val, const u32 *mask, unsigned int nbits,
                        ethnl_string_array_t names, bool compact)
{
        unsigned int len = 0;

        /* list flag */
        if (!mask)
                len += nla_total_size(sizeof(u32));
        /* size */
        len += nla_total_size(sizeof(u32));

        if (compact) {
                unsigned int nwords = DIV_ROUND_UP(nbits, 32);

                /* value, mask */
                len += (mask ? 2 : 1) * nla_total_size(nwords * sizeof(u32));
        } else {
                unsigned int bits_len = 0;
                unsigned int bit_len, i;

                for (i = 0; i < nbits; i++) {
                        const char *name = names ? names[i] : NULL;

                        if (!ethnl_bitmap32_test_bit(mask ?: val, i))
                                continue;
                        /* index */
                        bit_len = nla_total_size(sizeof(u32));
                        /* name */
                        if (name)
                                bit_len += ethnl_strz_size(name);
                        /* value */
                        if (mask && ethnl_bitmap32_test_bit(val, i))
                                bit_len += nla_total_size(0);

                        /* bit nest */
                        bits_len += nla_total_size(bit_len);
                }
                /* bits nest */
                len += nla_total_size(bits_len);
        }

        /* outermost nest */
        return nla_total_size(len);
}

/**
 * ethnl_put_bitset32() - Put a bitset nest into a message
 * @skb:      skb with the message
 * @attrtype: attribute type for the bitset nest
 * @val:      value bitmap (u32 based)
 * @mask:     mask bitmap (u32 based, optional)
 * @nbits:    bit length of the bitset
 * @names:    array of bit names (optional)
 * @compact:  use compact format for the output
 *
 * Compose a nested attribute representing a bitset. If @mask is null, simple
 * bitmap (bit list) is created, if @mask is provided, represent a value/mask
 * pair. Bit names are only used in verbose mode and when provided by calller.
 *
 * Return: 0 on success, negative error value on error
 */
int ethnl_put_bitset32(struct sk_buff *skb, int attrtype, const u32 *val,
                       const u32 *mask, unsigned int nbits,
                       ethnl_string_array_t names, bool compact)
{
        struct nlattr *nest;
        struct nlattr *attr;

        nest = nla_nest_start(skb, attrtype);
        if (!nest)
                return -EMSGSIZE;

        if (!mask && nla_put_flag(skb, ETHTOOL_A_BITSET_NOMASK))
                goto nla_put_failure;
        if (nla_put_u32(skb, ETHTOOL_A_BITSET_SIZE, nbits))
                goto nla_put_failure;
        if (compact) {
                unsigned int nwords = DIV_ROUND_UP(nbits, 32);
                unsigned int nbytes = nwords * sizeof(u32);
                u32 *dst;

                attr = nla_reserve(skb, ETHTOOL_A_BITSET_VALUE, nbytes);
                if (!attr)
                        goto nla_put_failure;
                dst = nla_data(attr);
                memcpy(dst, val, nbytes);
                if (nbits % 32)
                        dst[nwords - 1] &= ethnl_lower_bits(nbits);

                if (mask) {
                        attr = nla_reserve(skb, ETHTOOL_A_BITSET_MASK, nbytes);
                        if (!attr)
                                goto nla_put_failure;
                        dst = nla_data(attr);
                        memcpy(dst, mask, nbytes);
                        if (nbits % 32)
                                dst[nwords - 1] &= ethnl_lower_bits(nbits);
                }
        } else {
                struct nlattr *bits;
                unsigned int i;

                bits = nla_nest_start(skb, ETHTOOL_A_BITSET_BITS);
                if (!bits)
                        goto nla_put_failure;
                for (i = 0; i < nbits; i++) {
                        const char *name = names ? names[i] : NULL;

                        if (!ethnl_bitmap32_test_bit(mask ?: val, i))
                                continue;
                        attr = nla_nest_start(skb, ETHTOOL_A_BITSET_BITS_BIT);
                        if (!attr)
                                goto nla_put_failure;
                        if (nla_put_u32(skb, ETHTOOL_A_BITSET_BIT_INDEX, i))
                                goto nla_put_failure;
                        if (name &&
                            ethnl_put_strz(skb, ETHTOOL_A_BITSET_BIT_NAME, name))
                                goto nla_put_failure;
                        if (mask && ethnl_bitmap32_test_bit(val, i) &&
                            nla_put_flag(skb, ETHTOOL_A_BITSET_BIT_VALUE))
                                goto nla_put_failure;
                        nla_nest_end(skb, attr);
                }
                nla_nest_end(skb, bits);
        }

        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static const struct nla_policy bitset_policy[] = {
        [ETHTOOL_A_BITSET_NOMASK]        = { .type = NLA_FLAG },
        [ETHTOOL_A_BITSET_SIZE]                = NLA_POLICY_MAX(NLA_U32,
                                                         ETHNL_MAX_BITSET_SIZE),
        [ETHTOOL_A_BITSET_BITS]                = { .type = NLA_NESTED },
        [ETHTOOL_A_BITSET_VALUE]        = { .type = NLA_BINARY },
        [ETHTOOL_A_BITSET_MASK]                = { .type = NLA_BINARY },
};

static const struct nla_policy bit_policy[] = {
        [ETHTOOL_A_BITSET_BIT_INDEX]        = { .type = NLA_U32 },
        [ETHTOOL_A_BITSET_BIT_NAME]        = { .type = NLA_NUL_STRING },
        [ETHTOOL_A_BITSET_BIT_VALUE]        = { .type = NLA_FLAG },
};

/**
 * ethnl_bitset_is_compact() - check if bitset attribute represents a compact
 *                               bitset
 * @bitset:  nested attribute representing a bitset
 * @compact: pointer for return value
 *
 * Return: 0 on success, negative error code on failure
 */
int ethnl_bitset_is_compact(const struct nlattr *bitset, bool *compact)
{
        struct nlattr *tb[ARRAY_SIZE(bitset_policy)];
        int ret;

        ret = nla_parse_nested(tb, ARRAY_SIZE(bitset_policy) - 1, bitset,
                               bitset_policy, NULL);
        if (ret < 0)
                return ret;

        if (tb[ETHTOOL_A_BITSET_BITS]) {
                if (tb[ETHTOOL_A_BITSET_VALUE] || tb[ETHTOOL_A_BITSET_MASK])
                        return -EINVAL;
                *compact = false;
                return 0;
        }
        if (!tb[ETHTOOL_A_BITSET_SIZE] || !tb[ETHTOOL_A_BITSET_VALUE])
                return -EINVAL;

        *compact = true;
        return 0;
}

/**
 * ethnl_name_to_idx() - look up string index for a name
 * @names:   array of ETH_GSTRING_LEN sized strings
 * @n_names: number of strings in the array
 * @name:    name to look up
 *
 * Return: index of the string if found, -ENOENT if not found
 */
static int ethnl_name_to_idx(ethnl_string_array_t names, unsigned int n_names,
                             const char *name)
{
        unsigned int i;

        if (!names)
                return -ENOENT;

        for (i = 0; i < n_names; i++) {
                /* names[i] may not be null terminated */
                if (!strncmp(names[i], name, ETH_GSTRING_LEN) &&
                    strlen(name) <= ETH_GSTRING_LEN)
                        return i;
        }

        return -ENOENT;
}

static int ethnl_parse_bit(unsigned int *index, bool *val, unsigned int nbits,
                           const struct nlattr *bit_attr, bool no_mask,
                           ethnl_string_array_t names,
                           struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(bit_policy)];
        int ret, idx;

        ret = nla_parse_nested(tb, ARRAY_SIZE(bit_policy) - 1, bit_attr,
                               bit_policy, extack);
        if (ret < 0)
                return ret;

        if (tb[ETHTOOL_A_BITSET_BIT_INDEX]) {
                const char *name;

                idx = nla_get_u32(tb[ETHTOOL_A_BITSET_BIT_INDEX]);
                if (idx >= nbits) {
                        NL_SET_ERR_MSG_ATTR(extack,
                                            tb[ETHTOOL_A_BITSET_BIT_INDEX],
                                            "bit index too high");
                        return -EOPNOTSUPP;
                }
                name = names ? names[idx] : NULL;
                if (tb[ETHTOOL_A_BITSET_BIT_NAME] && name &&
                    strncmp(nla_data(tb[ETHTOOL_A_BITSET_BIT_NAME]), name,
                            nla_len(tb[ETHTOOL_A_BITSET_BIT_NAME]))) {
                        NL_SET_ERR_MSG_ATTR(extack, bit_attr,
                                            "bit index and name mismatch");
                        return -EINVAL;
                }
        } else if (tb[ETHTOOL_A_BITSET_BIT_NAME]) {
                idx = ethnl_name_to_idx(names, nbits,
                                        nla_data(tb[ETHTOOL_A_BITSET_BIT_NAME]));
                if (idx < 0) {
                        NL_SET_ERR_MSG_ATTR(extack,
                                            tb[ETHTOOL_A_BITSET_BIT_NAME],
                                            "bit name not found");
                        return -EOPNOTSUPP;
                }
        } else {
                NL_SET_ERR_MSG_ATTR(extack, bit_attr,
                                    "neither bit index nor name specified");
                return -EINVAL;
        }

        *index = idx;
        *val = no_mask || tb[ETHTOOL_A_BITSET_BIT_VALUE];
        return 0;
}

/**
 * ethnl_bitmap32_equal() - Compare two bitmaps
 * @map1:  first bitmap
 * @map2:  second bitmap
 * @nbits: bit size to compare
 *
 * Return: true if first @nbits are equal, false if not
 */
static bool ethnl_bitmap32_equal(const u32 *map1, const u32 *map2,
                                 unsigned int nbits)
{
        if (memcmp(map1, map2, nbits / 32 * sizeof(u32)))
                return false;
        if (nbits % 32 == 0)
                return true;
        return !((map1[nbits / 32] ^ map2[nbits / 32]) &
                 ethnl_lower_bits(nbits % 32));
}

static int
ethnl_update_bitset32_verbose(u32 *bitmap, unsigned int nbits,
                              const struct nlattr *attr, struct nlattr **tb,
                              ethnl_string_array_t names,
                              struct netlink_ext_ack *extack, bool *mod)
{
        u32 *saved_bitmap = NULL;
        struct nlattr *bit_attr;
        bool no_mask;
        int rem;
        int ret;

        if (tb[ETHTOOL_A_BITSET_VALUE]) {
                NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_BITSET_VALUE],
                                    "value only allowed in compact bitset");
                return -EINVAL;
        }
        if (tb[ETHTOOL_A_BITSET_MASK]) {
                NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_BITSET_MASK],
                                    "mask only allowed in compact bitset");
                return -EINVAL;
        }

        no_mask = tb[ETHTOOL_A_BITSET_NOMASK];
        if (no_mask) {
                unsigned int nwords = DIV_ROUND_UP(nbits, 32);
                unsigned int nbytes = nwords * sizeof(u32);
                bool dummy;

                /* The bitmap size is only the size of the map part without
                 * its mask part.
                 */
                saved_bitmap = kcalloc(nwords, sizeof(u32), GFP_KERNEL);
                if (!saved_bitmap)
                        return -ENOMEM;
                memcpy(saved_bitmap, bitmap, nbytes);
                ethnl_bitmap32_clear(bitmap, 0, nbits, &dummy);
        }

        nla_for_each_nested(bit_attr, tb[ETHTOOL_A_BITSET_BITS], rem) {
                bool old_val, new_val;
                unsigned int idx;

                if (nla_type(bit_attr) != ETHTOOL_A_BITSET_BITS_BIT) {
                        NL_SET_ERR_MSG_ATTR(extack, bit_attr,
                                            "only ETHTOOL_A_BITSET_BITS_BIT allowed in ETHTOOL_A_BITSET_BITS");
                        kfree(saved_bitmap);
                        return -EINVAL;
                }
                ret = ethnl_parse_bit(&idx, &new_val, nbits, bit_attr, no_mask,
                                      names, extack);
                if (ret < 0) {
                        kfree(saved_bitmap);
                        return ret;
                }
                old_val = bitmap[idx / 32] & ((u32)1 << (idx % 32));
                if (new_val != old_val) {
                        if (new_val)
                                bitmap[idx / 32] |= ((u32)1 << (idx % 32));
                        else
                                bitmap[idx / 32] &= ~((u32)1 << (idx % 32));
                        if (!no_mask)
                                *mod = true;
                }
        }

        if (no_mask && !ethnl_bitmap32_equal(saved_bitmap, bitmap, nbits))
                *mod = true;

        kfree(saved_bitmap);
        return 0;
}

static int ethnl_compact_sanity_checks(unsigned int nbits,
                                       const struct nlattr *nest,
                                       struct nlattr **tb,
                                       struct netlink_ext_ack *extack)
{
        bool no_mask = tb[ETHTOOL_A_BITSET_NOMASK];
        unsigned int attr_nbits, attr_nwords;
        const struct nlattr *test_attr;

        if (no_mask && tb[ETHTOOL_A_BITSET_MASK]) {
                NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_BITSET_MASK],
                                    "mask not allowed in list bitset");
                return -EINVAL;
        }
        if (!tb[ETHTOOL_A_BITSET_SIZE]) {
                NL_SET_ERR_MSG_ATTR(extack, nest,
                                    "missing size in compact bitset");
                return -EINVAL;
        }
        if (!tb[ETHTOOL_A_BITSET_VALUE]) {
                NL_SET_ERR_MSG_ATTR(extack, nest,
                                    "missing value in compact bitset");
                return -EINVAL;
        }
        if (!no_mask && !tb[ETHTOOL_A_BITSET_MASK]) {
                NL_SET_ERR_MSG_ATTR(extack, nest,
                                    "missing mask in compact nonlist bitset");
                return -EINVAL;
        }

        attr_nbits = nla_get_u32(tb[ETHTOOL_A_BITSET_SIZE]);
        attr_nwords = DIV_ROUND_UP(attr_nbits, 32);
        if (nla_len(tb[ETHTOOL_A_BITSET_VALUE]) != attr_nwords * sizeof(u32)) {
                NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_BITSET_VALUE],
                                    "bitset value length does not match size");
                return -EINVAL;
        }
        if (tb[ETHTOOL_A_BITSET_MASK] &&
            nla_len(tb[ETHTOOL_A_BITSET_MASK]) != attr_nwords * sizeof(u32)) {
                NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_BITSET_MASK],
                                    "bitset mask length does not match size");
                return -EINVAL;
        }
        if (attr_nbits <= nbits)
                return 0;

        test_attr = no_mask ? tb[ETHTOOL_A_BITSET_VALUE] :
                              tb[ETHTOOL_A_BITSET_MASK];
        if (ethnl_bitmap32_not_zero(nla_data(test_attr), nbits, attr_nbits)) {
                NL_SET_ERR_MSG_ATTR(extack, test_attr,
                                    "cannot modify bits past kernel bitset size");
                return -EINVAL;
        }
        return 0;
}

/**
 * ethnl_update_bitset32() - Apply a bitset nest to a u32 based bitmap
 * @bitmap:  bitmap to update
 * @nbits:   size of the updated bitmap in bits
 * @attr:    nest attribute to parse and apply
 * @names:   array of bit names; may be null for compact format
 * @extack:  extack for error reporting
 * @mod:     set this to true if bitmap is modified, leave as it is if not
 *
 * Apply bitset netsted attribute to a bitmap. If the attribute represents
 * a bit list, @bitmap is set to its contents; otherwise, bits in mask are
 * set to values from value. Bitmaps in the attribute may be longer than
 * @nbits but the message must not request modifying any bits past @nbits.
 *
 * Return: negative error code on failure, 0 on success
 */
int ethnl_update_bitset32(u32 *bitmap, unsigned int nbits,
                          const struct nlattr *attr, ethnl_string_array_t names,
                          struct netlink_ext_ack *extack, bool *mod)
{
        struct nlattr *tb[ARRAY_SIZE(bitset_policy)];
        unsigned int change_bits;
        bool no_mask;
        int ret;

        if (!attr)
                return 0;
        ret = nla_parse_nested(tb, ARRAY_SIZE(bitset_policy) - 1, attr,
                               bitset_policy, extack);
        if (ret < 0)
                return ret;

        if (tb[ETHTOOL_A_BITSET_BITS])
                return ethnl_update_bitset32_verbose(bitmap, nbits, attr, tb,
                                                     names, extack, mod);
        ret = ethnl_compact_sanity_checks(nbits, attr, tb, extack);
        if (ret < 0)
                return ret;

        no_mask = tb[ETHTOOL_A_BITSET_NOMASK];
        change_bits = min_t(unsigned int,
                            nla_get_u32(tb[ETHTOOL_A_BITSET_SIZE]), nbits);
        ethnl_bitmap32_update(bitmap, change_bits,
                              nla_data(tb[ETHTOOL_A_BITSET_VALUE]),
                              no_mask ? NULL :
                                        nla_data(tb[ETHTOOL_A_BITSET_MASK]),
                              mod);
        if (no_mask && change_bits < nbits)
                ethnl_bitmap32_clear(bitmap, change_bits, nbits, mod);

        return 0;
}

/**
 * ethnl_parse_bitset() - Compute effective value and mask from bitset nest
 * @val:     unsigned long based bitmap to put value into
 * @mask:    unsigned long based bitmap to put mask into
 * @nbits:   size of @val and @mask bitmaps
 * @attr:    nest attribute to parse and apply
 * @names:   array of bit names; may be null for compact format
 * @extack:  extack for error reporting
 *
 * Provide @nbits size long bitmaps for value and mask so that
 * x = (val & mask) | (x & ~mask) would modify any @nbits sized bitmap x
 * the same way ethnl_update_bitset() with the same bitset attribute would.
 *
 * Return:   negative error code on failure, 0 on success
 */
int ethnl_parse_bitset(unsigned long *val, unsigned long *mask,
                       unsigned int nbits, const struct nlattr *attr,
                       ethnl_string_array_t names,
                       struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(bitset_policy)];
        const struct nlattr *bit_attr;
        bool no_mask;
        int rem;
        int ret;

        if (!attr)
                return 0;
        ret = nla_parse_nested(tb, ARRAY_SIZE(bitset_policy) - 1, attr,
                               bitset_policy, extack);
        if (ret < 0)
                return ret;
        no_mask = tb[ETHTOOL_A_BITSET_NOMASK];

        if (!tb[ETHTOOL_A_BITSET_BITS]) {
                unsigned int change_bits;

                ret = ethnl_compact_sanity_checks(nbits, attr, tb, extack);
                if (ret < 0)
                        return ret;

                change_bits = nla_get_u32(tb[ETHTOOL_A_BITSET_SIZE]);
                if (change_bits > nbits)
                        change_bits = nbits;
                bitmap_from_arr32(val, nla_data(tb[ETHTOOL_A_BITSET_VALUE]),
                                  change_bits);
                if (change_bits < nbits)
                        bitmap_clear(val, change_bits, nbits - change_bits);
                if (no_mask) {
                        bitmap_fill(mask, nbits);
                } else {
                        bitmap_from_arr32(mask,
                                          nla_data(tb[ETHTOOL_A_BITSET_MASK]),
                                          change_bits);
                        if (change_bits < nbits)
                                bitmap_clear(mask, change_bits,
                                             nbits - change_bits);
                }

                return 0;
        }

        if (tb[ETHTOOL_A_BITSET_VALUE]) {
                NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_BITSET_VALUE],
                                    "value only allowed in compact bitset");
                return -EINVAL;
        }
        if (tb[ETHTOOL_A_BITSET_MASK]) {
                NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_BITSET_MASK],
                                    "mask only allowed in compact bitset");
                return -EINVAL;
        }

        bitmap_zero(val, nbits);
        if (no_mask)
                bitmap_fill(mask, nbits);
        else
                bitmap_zero(mask, nbits);

        nla_for_each_nested(bit_attr, tb[ETHTOOL_A_BITSET_BITS], rem) {
                unsigned int idx;
                bool bit_val;

                ret = ethnl_parse_bit(&idx, &bit_val, nbits, bit_attr, no_mask,
                                      names, extack);
                if (ret < 0)
                        return ret;
                if (bit_val)
                        __set_bit(idx, val);
                if (!no_mask)
                        __set_bit(idx, mask);
        }

        return 0;
}

#if BITS_PER_LONG == 64 && defined(__BIG_ENDIAN)

/* 64-bit big endian architectures are the only case when u32 based bitmaps
 * and unsigned long based bitmaps have different memory layout so that we
 * cannot simply cast the latter to the former and need actual wrappers
 * converting the latter to the former.
 *
 * To reduce the number of slab allocations, the wrappers use fixed size local
 * variables for bitmaps up to ETHNL_SMALL_BITMAP_BITS bits which is the
 * majority of bitmaps used by ethtool.
 */
#define ETHNL_SMALL_BITMAP_BITS 128
#define ETHNL_SMALL_BITMAP_WORDS DIV_ROUND_UP(ETHNL_SMALL_BITMAP_BITS, 32)

int ethnl_bitset_size(const unsigned long *val, const unsigned long *mask,
                      unsigned int nbits, ethnl_string_array_t names,
                      bool compact)
{
        u32 small_mask32[ETHNL_SMALL_BITMAP_WORDS];
        u32 small_val32[ETHNL_SMALL_BITMAP_WORDS];
        u32 *mask32;
        u32 *val32;
        int ret;

        if (nbits > ETHNL_SMALL_BITMAP_BITS) {
                unsigned int nwords = DIV_ROUND_UP(nbits, 32);

                val32 = kmalloc_array(2 * nwords, sizeof(u32), GFP_KERNEL);
                if (!val32)
                        return -ENOMEM;
                mask32 = val32 + nwords;
        } else {
                val32 = small_val32;
                mask32 = small_mask32;
        }

        bitmap_to_arr32(val32, val, nbits);
        if (mask)
                bitmap_to_arr32(mask32, mask, nbits);
        else
                mask32 = NULL;
        ret = ethnl_bitset32_size(val32, mask32, nbits, names, compact);

        if (nbits > ETHNL_SMALL_BITMAP_BITS)
                kfree(val32);

        return ret;
}

int ethnl_put_bitset(struct sk_buff *skb, int attrtype,
                     const unsigned long *val, const unsigned long *mask,
                     unsigned int nbits, ethnl_string_array_t names,
                     bool compact)
{
        u32 small_mask32[ETHNL_SMALL_BITMAP_WORDS];
        u32 small_val32[ETHNL_SMALL_BITMAP_WORDS];
        u32 *mask32;
        u32 *val32;
        int ret;

        if (nbits > ETHNL_SMALL_BITMAP_BITS) {
                unsigned int nwords = DIV_ROUND_UP(nbits, 32);

                val32 = kmalloc_array(2 * nwords, sizeof(u32), GFP_KERNEL);
                if (!val32)
                        return -ENOMEM;
                mask32 = val32 + nwords;
        } else {
                val32 = small_val32;
                mask32 = small_mask32;
        }

        bitmap_to_arr32(val32, val, nbits);
        if (mask)
                bitmap_to_arr32(mask32, mask, nbits);
        else
                mask32 = NULL;
        ret = ethnl_put_bitset32(skb, attrtype, val32, mask32, nbits, names,
                                 compact);

        if (nbits > ETHNL_SMALL_BITMAP_BITS)
                kfree(val32);

        return ret;
}

int ethnl_update_bitset(unsigned long *bitmap, unsigned int nbits,
                        const struct nlattr *attr, ethnl_string_array_t names,
                        struct netlink_ext_ack *extack, bool *mod)
{
        u32 small_bitmap32[ETHNL_SMALL_BITMAP_WORDS];
        u32 *bitmap32 = small_bitmap32;
        bool u32_mod = false;
        int ret;

        if (nbits > ETHNL_SMALL_BITMAP_BITS) {
                unsigned int dst_words = DIV_ROUND_UP(nbits, 32);

                bitmap32 = kmalloc_array(dst_words, sizeof(u32), GFP_KERNEL);
                if (!bitmap32)
                        return -ENOMEM;
        }

        bitmap_to_arr32(bitmap32, bitmap, nbits);
        ret = ethnl_update_bitset32(bitmap32, nbits, attr, names, extack,
                                    &u32_mod);
        if (u32_mod) {
                bitmap_from_arr32(bitmap, bitmap32, nbits);
                *mod = true;
        }

        if (nbits > ETHNL_SMALL_BITMAP_BITS)
                kfree(bitmap32);

        return ret;
}

#else

/* On little endian 64-bit and all 32-bit architectures, an unsigned long
 * based bitmap can be interpreted as u32 based one using a simple cast.
 */

int ethnl_bitset_size(const unsigned long *val, const unsigned long *mask,
                      unsigned int nbits, ethnl_string_array_t names,
                      bool compact)
{
        return ethnl_bitset32_size((const u32 *)val, (const u32 *)mask, nbits,
                                   names, compact);
}

int ethnl_put_bitset(struct sk_buff *skb, int attrtype,
                     const unsigned long *val, const unsigned long *mask,
                     unsigned int nbits, ethnl_string_array_t names,
                     bool compact)
{
        return ethnl_put_bitset32(skb, attrtype, (const u32 *)val,
                                  (const u32 *)mask, nbits, names, compact);
}

int ethnl_update_bitset(unsigned long *bitmap, unsigned int nbits,
                        const struct nlattr *attr, ethnl_string_array_t names,
                        struct netlink_ext_ack *extack, bool *mod)
{
        return ethnl_update_bitset32((u32 *)bitmap, nbits, attr, names, extack,
                                     mod);
}

#endif /* BITS_PER_LONG == 64 && defined(__BIG_ENDIAN) */




































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   68 

































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/socket.h>
#include <linux/skbuff.h>
#include <linux/ip.h>
#include <linux/icmp.h>
#include <linux/udp.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <net/genetlink.h>
#include <net/gro.h>
#include <net/gue.h>
#include <net/fou.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/udp.h>
#include <net/udp_tunnel.h>
#include <uapi/linux/fou.h>
#include <uapi/linux/genetlink.h>

#include "fou_nl.h"

struct fou {
        struct socket *sock;
        u8 protocol;
        u8 flags;
        __be16 port;
        u8 family;
        u16 type;
        struct list_head list;
        struct rcu_head rcu;
};

#define FOU_F_REMCSUM_NOPARTIAL BIT(0)

struct fou_cfg {
        u16 type;
        u8 protocol;
        u8 flags;
        struct udp_port_cfg udp_config;
};

static unsigned int fou_net_id;

struct fou_net {
        struct list_head fou_list;
        struct mutex fou_lock;
};

static inline struct fou *fou_from_sock(struct sock *sk)
{
        return rcu_dereference_sk_user_data(sk);
}

static int fou_recv_pull(struct sk_buff *skb, struct fou *fou, size_t len)
{
        /* Remove 'len' bytes from the packet (UDP header and
         * FOU header if present).
         */
        if (fou->family == AF_INET)
                ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len);
        else
                ipv6_hdr(skb)->payload_len =
                    htons(ntohs(ipv6_hdr(skb)->payload_len) - len);

        __skb_pull(skb, len);
        skb_postpull_rcsum(skb, udp_hdr(skb), len);
        skb_reset_transport_header(skb);
        return iptunnel_pull_offloads(skb);
}

static int fou_udp_recv(struct sock *sk, struct sk_buff *skb)
{
        struct fou *fou = fou_from_sock(sk);

        if (!fou)
                return 1;

        if (fou_recv_pull(skb, fou, sizeof(struct udphdr)))
                goto drop;

        return -fou->protocol;

drop:
        kfree_skb(skb);
        return 0;
}

static struct guehdr *gue_remcsum(struct sk_buff *skb, struct guehdr *guehdr,
                                  void *data, size_t hdrlen, u8 ipproto,
                                  bool nopartial)
{
        __be16 *pd = data;
        size_t start = ntohs(pd[0]);
        size_t offset = ntohs(pd[1]);
        size_t plen = sizeof(struct udphdr) + hdrlen +
            max_t(size_t, offset + sizeof(u16), start);

        if (skb->remcsum_offload)
                return guehdr;

        if (!pskb_may_pull(skb, plen))
                return NULL;
        guehdr = (struct guehdr *)&udp_hdr(skb)[1];

        skb_remcsum_process(skb, (void *)guehdr + hdrlen,
                            start, offset, nopartial);

        return guehdr;
}

static int gue_control_message(struct sk_buff *skb, struct guehdr *guehdr)
{
        /* No support yet */
        kfree_skb(skb);
        return 0;
}

static int gue_udp_recv(struct sock *sk, struct sk_buff *skb)
{
        struct fou *fou = fou_from_sock(sk);
        size_t len, optlen, hdrlen;
        struct guehdr *guehdr;
        void *data;
        u16 doffset = 0;
        u8 proto_ctype;

        if (!fou)
                return 1;

        len = sizeof(struct udphdr) + sizeof(struct guehdr);
        if (!pskb_may_pull(skb, len))
                goto drop;

        guehdr = (struct guehdr *)&udp_hdr(skb)[1];

        switch (guehdr->version) {
        case 0: /* Full GUE header present */
                break;

        case 1: {
                /* Direct encapsulation of IPv4 or IPv6 */

                int prot;

                switch (((struct iphdr *)guehdr)->version) {
                case 4:
                        prot = IPPROTO_IPIP;
                        break;
                case 6:
                        prot = IPPROTO_IPV6;
                        break;
                default:
                        goto drop;
                }

                if (fou_recv_pull(skb, fou, sizeof(struct udphdr)))
                        goto drop;

                return -prot;
        }

        default: /* Undefined version */
                goto drop;
        }

        optlen = guehdr->hlen << 2;
        len += optlen;

        if (!pskb_may_pull(skb, len))
                goto drop;

        /* guehdr may change after pull */
        guehdr = (struct guehdr *)&udp_hdr(skb)[1];

        if (validate_gue_flags(guehdr, optlen))
                goto drop;

        hdrlen = sizeof(struct guehdr) + optlen;

        if (fou->family == AF_INET)
                ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len);
        else
                ipv6_hdr(skb)->payload_len =
                    htons(ntohs(ipv6_hdr(skb)->payload_len) - len);

        /* Pull csum through the guehdr now . This can be used if
         * there is a remote checksum offload.
         */
        skb_postpull_rcsum(skb, udp_hdr(skb), len);

        data = &guehdr[1];

        if (guehdr->flags & GUE_FLAG_PRIV) {
                __be32 flags = *(__be32 *)(data + doffset);

                doffset += GUE_LEN_PRIV;

                if (flags & GUE_PFLAG_REMCSUM) {
                        guehdr = gue_remcsum(skb, guehdr, data + doffset,
                                             hdrlen, guehdr->proto_ctype,
                                             !!(fou->flags &
                                                FOU_F_REMCSUM_NOPARTIAL));
                        if (!guehdr)
                                goto drop;

                        data = &guehdr[1];

                        doffset += GUE_PLEN_REMCSUM;
                }
        }

        if (unlikely(guehdr->control))
                return gue_control_message(skb, guehdr);

        proto_ctype = guehdr->proto_ctype;
        __skb_pull(skb, sizeof(struct udphdr) + hdrlen);
        skb_reset_transport_header(skb);

        if (iptunnel_pull_offloads(skb))
                goto drop;

        return -proto_ctype;

drop:
        kfree_skb(skb);
        return 0;
}

static struct sk_buff *fou_gro_receive(struct sock *sk,
                                       struct list_head *head,
                                       struct sk_buff *skb)
{
        const struct net_offload __rcu **offloads;
        struct fou *fou = fou_from_sock(sk);
        const struct net_offload *ops;
        struct sk_buff *pp = NULL;
        u8 proto;

        if (!fou)
                goto out;

        proto = fou->protocol;

        /* We can clear the encap_mark for FOU as we are essentially doing
         * one of two possible things.  We are either adding an L4 tunnel
         * header to the outer L3 tunnel header, or we are simply
         * treating the GRE tunnel header as though it is a UDP protocol
         * specific header such as VXLAN or GENEVE.
         */
        NAPI_GRO_CB(skb)->encap_mark = 0;

        /* Flag this frame as already having an outer encap header */
        NAPI_GRO_CB(skb)->is_fou = 1;

        offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
        ops = rcu_dereference(offloads[proto]);
        if (!ops || !ops->callbacks.gro_receive)
                goto out;

        pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);

out:
        return pp;
}

static int fou_gro_complete(struct sock *sk, struct sk_buff *skb,
                            int nhoff)
{
        const struct net_offload __rcu **offloads;
        struct fou *fou = fou_from_sock(sk);
        const struct net_offload *ops;
        u8 proto;
        int err;

        if (!fou) {
                err = -ENOENT;
                goto out;
        }

        proto = fou->protocol;

        offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
        ops = rcu_dereference(offloads[proto]);
        if (WARN_ON(!ops || !ops->callbacks.gro_complete)) {
                err = -ENOSYS;
                goto out;
        }

        err = ops->callbacks.gro_complete(skb, nhoff);

        skb_set_inner_mac_header(skb, nhoff);

out:
        return err;
}

static struct guehdr *gue_gro_remcsum(struct sk_buff *skb, unsigned int off,
                                      struct guehdr *guehdr, void *data,
                                      size_t hdrlen, struct gro_remcsum *grc,
                                      bool nopartial)
{
        __be16 *pd = data;
        size_t start = ntohs(pd[0]);
        size_t offset = ntohs(pd[1]);

        if (skb->remcsum_offload)
                return guehdr;

        if (!NAPI_GRO_CB(skb)->csum_valid)
                return NULL;

        guehdr = skb_gro_remcsum_process(skb, (void *)guehdr, off, hdrlen,
                                         start, offset, grc, nopartial);

        skb->remcsum_offload = 1;

        return guehdr;
}

static struct sk_buff *gue_gro_receive(struct sock *sk,
                                       struct list_head *head,
                                       struct sk_buff *skb)
{
        const struct net_offload __rcu **offloads;
        const struct net_offload *ops;
        struct sk_buff *pp = NULL;
        struct sk_buff *p;
        struct guehdr *guehdr;
        size_t len, optlen, hdrlen, off;
        void *data;
        u16 doffset = 0;
        int flush = 1;
        struct fou *fou = fou_from_sock(sk);
        struct gro_remcsum grc;
        u8 proto;

        skb_gro_remcsum_init(&grc);

        if (!fou)
                goto out;

        off = skb_gro_offset(skb);
        len = off + sizeof(*guehdr);

        guehdr = skb_gro_header(skb, len, off);
        if (unlikely(!guehdr))
                goto out;

        switch (guehdr->version) {
        case 0:
                break;
        case 1:
                switch (((struct iphdr *)guehdr)->version) {
                case 4:
                        proto = IPPROTO_IPIP;
                        break;
                case 6:
                        proto = IPPROTO_IPV6;
                        break;
                default:
                        goto out;
                }
                goto next_proto;
        default:
                goto out;
        }

        optlen = guehdr->hlen << 2;
        len += optlen;

        if (!skb_gro_may_pull(skb, len)) {
                guehdr = skb_gro_header_slow(skb, len, off);
                if (unlikely(!guehdr))
                        goto out;
        }

        if (unlikely(guehdr->control) || guehdr->version != 0 ||
            validate_gue_flags(guehdr, optlen))
                goto out;

        hdrlen = sizeof(*guehdr) + optlen;

        /* Adjust NAPI_GRO_CB(skb)->csum to account for guehdr,
         * this is needed if there is a remote checkcsum offload.
         */
        skb_gro_postpull_rcsum(skb, guehdr, hdrlen);

        data = &guehdr[1];

        if (guehdr->flags & GUE_FLAG_PRIV) {
                __be32 flags = *(__be32 *)(data + doffset);

                doffset += GUE_LEN_PRIV;

                if (flags & GUE_PFLAG_REMCSUM) {
                        guehdr = gue_gro_remcsum(skb, off, guehdr,
                                                 data + doffset, hdrlen, &grc,
                                                 !!(fou->flags &
                                                    FOU_F_REMCSUM_NOPARTIAL));

                        if (!guehdr)
                                goto out;

                        data = &guehdr[1];

                        doffset += GUE_PLEN_REMCSUM;
                }
        }

        skb_gro_pull(skb, hdrlen);

        list_for_each_entry(p, head, list) {
                const struct guehdr *guehdr2;

                if (!NAPI_GRO_CB(p)->same_flow)
                        continue;

                guehdr2 = (struct guehdr *)(p->data + off);

                /* Compare base GUE header to be equal (covers
                 * hlen, version, proto_ctype, and flags.
                 */
                if (guehdr->word != guehdr2->word) {
                        NAPI_GRO_CB(p)->same_flow = 0;
                        continue;
                }

                /* Compare optional fields are the same. */
                if (guehdr->hlen && memcmp(&guehdr[1], &guehdr2[1],
                                           guehdr->hlen << 2)) {
                        NAPI_GRO_CB(p)->same_flow = 0;
                        continue;
                }
        }

        proto = guehdr->proto_ctype;

next_proto:

        /* We can clear the encap_mark for GUE as we are essentially doing
         * one of two possible things.  We are either adding an L4 tunnel
         * header to the outer L3 tunnel header, or we are simply
         * treating the GRE tunnel header as though it is a UDP protocol
         * specific header such as VXLAN or GENEVE.
         */
        NAPI_GRO_CB(skb)->encap_mark = 0;

        /* Flag this frame as already having an outer encap header */
        NAPI_GRO_CB(skb)->is_fou = 1;

        offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
        ops = rcu_dereference(offloads[proto]);
        if (!ops || !ops->callbacks.gro_receive)
                goto out;

        pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
        flush = 0;

out:
        skb_gro_flush_final_remcsum(skb, pp, flush, &grc);

        return pp;
}

static int gue_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff)
{
        struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff);
        const struct net_offload __rcu **offloads;
        const struct net_offload *ops;
        unsigned int guehlen = 0;
        u8 proto;
        int err = -ENOENT;

        switch (guehdr->version) {
        case 0:
                proto = guehdr->proto_ctype;
                guehlen = sizeof(*guehdr) + (guehdr->hlen << 2);
                break;
        case 1:
                switch (((struct iphdr *)guehdr)->version) {
                case 4:
                        proto = IPPROTO_IPIP;
                        break;
                case 6:
                        proto = IPPROTO_IPV6;
                        break;
                default:
                        return err;
                }
                break;
        default:
                return err;
        }

        offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
        ops = rcu_dereference(offloads[proto]);
        if (WARN_ON(!ops || !ops->callbacks.gro_complete))
                goto out;

        err = ops->callbacks.gro_complete(skb, nhoff + guehlen);

        skb_set_inner_mac_header(skb, nhoff + guehlen);

out:
        return err;
}

static bool fou_cfg_cmp(struct fou *fou, struct fou_cfg *cfg)
{
        struct sock *sk = fou->sock->sk;
        struct udp_port_cfg *udp_cfg = &cfg->udp_config;

        if (fou->family != udp_cfg->family ||
            fou->port != udp_cfg->local_udp_port ||
            sk->sk_dport != udp_cfg->peer_udp_port ||
            sk->sk_bound_dev_if != udp_cfg->bind_ifindex)
                return false;

        if (fou->family == AF_INET) {
                if (sk->sk_rcv_saddr != udp_cfg->local_ip.s_addr ||
                    sk->sk_daddr != udp_cfg->peer_ip.s_addr)
                        return false;
                else
                        return true;
#if IS_ENABLED(CONFIG_IPV6)
        } else {
                if (ipv6_addr_cmp(&sk->sk_v6_rcv_saddr, &udp_cfg->local_ip6) ||
                    ipv6_addr_cmp(&sk->sk_v6_daddr, &udp_cfg->peer_ip6))
                        return false;
                else
                        return true;
#endif
        }

        return false;
}

static int fou_add_to_port_list(struct net *net, struct fou *fou,
                                struct fou_cfg *cfg)
{
        struct fou_net *fn = net_generic(net, fou_net_id);
        struct fou *fout;

        mutex_lock(&fn->fou_lock);
        list_for_each_entry(fout, &fn->fou_list, list) {
                if (fou_cfg_cmp(fout, cfg)) {
                        mutex_unlock(&fn->fou_lock);
                        return -EALREADY;
                }
        }

        list_add(&fou->list, &fn->fou_list);
        mutex_unlock(&fn->fou_lock);

        return 0;
}

static void fou_release(struct fou *fou)
{
        struct socket *sock = fou->sock;

        list_del(&fou->list);
        udp_tunnel_sock_release(sock);

        kfree_rcu(fou, rcu);
}

static int fou_create(struct net *net, struct fou_cfg *cfg,
                      struct socket **sockp)
{
        struct socket *sock = NULL;
        struct fou *fou = NULL;
        struct sock *sk;
        struct udp_tunnel_sock_cfg tunnel_cfg;
        int err;

        /* Open UDP socket */
        err = udp_sock_create(net, &cfg->udp_config, &sock);
        if (err < 0)
                goto error;

        /* Allocate FOU port structure */
        fou = kzalloc(sizeof(*fou), GFP_KERNEL);
        if (!fou) {
                err = -ENOMEM;
                goto error;
        }

        sk = sock->sk;

        fou->port = cfg->udp_config.local_udp_port;
        fou->family = cfg->udp_config.family;
        fou->flags = cfg->flags;
        fou->type = cfg->type;
        fou->sock = sock;

        memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
        tunnel_cfg.encap_type = 1;
        tunnel_cfg.sk_user_data = fou;
        tunnel_cfg.encap_destroy = NULL;

        /* Initial for fou type */
        switch (cfg->type) {
        case FOU_ENCAP_DIRECT:
                tunnel_cfg.encap_rcv = fou_udp_recv;
                tunnel_cfg.gro_receive = fou_gro_receive;
                tunnel_cfg.gro_complete = fou_gro_complete;
                fou->protocol = cfg->protocol;
                break;
        case FOU_ENCAP_GUE:
                tunnel_cfg.encap_rcv = gue_udp_recv;
                tunnel_cfg.gro_receive = gue_gro_receive;
                tunnel_cfg.gro_complete = gue_gro_complete;
                break;
        default:
                err = -EINVAL;
                goto error;
        }

        setup_udp_tunnel_sock(net, sock, &tunnel_cfg);

        sk->sk_allocation = GFP_ATOMIC;

        err = fou_add_to_port_list(net, fou, cfg);
        if (err)
                goto error;

        if (sockp)
                *sockp = sock;

        return 0;

error:
        kfree(fou);
        if (sock)
                udp_tunnel_sock_release(sock);

        return err;
}

static int fou_destroy(struct net *net, struct fou_cfg *cfg)
{
        struct fou_net *fn = net_generic(net, fou_net_id);
        int err = -EINVAL;
        struct fou *fou;

        mutex_lock(&fn->fou_lock);
        list_for_each_entry(fou, &fn->fou_list, list) {
                if (fou_cfg_cmp(fou, cfg)) {
                        fou_release(fou);
                        err = 0;
                        break;
                }
        }
        mutex_unlock(&fn->fou_lock);

        return err;
}

static struct genl_family fou_nl_family;

static int parse_nl_config(struct genl_info *info,
                           struct fou_cfg *cfg)
{
        bool has_local = false, has_peer = false;
        struct nlattr *attr;
        int ifindex;
        __be16 port;

        memset(cfg, 0, sizeof(*cfg));

        cfg->udp_config.family = AF_INET;

        if (info->attrs[FOU_ATTR_AF]) {
                u8 family = nla_get_u8(info->attrs[FOU_ATTR_AF]);

                switch (family) {
                case AF_INET:
                        break;
                case AF_INET6:
                        cfg->udp_config.ipv6_v6only = 1;
                        break;
                default:
                        return -EAFNOSUPPORT;
                }

                cfg->udp_config.family = family;
        }

        if (info->attrs[FOU_ATTR_PORT]) {
                port = nla_get_be16(info->attrs[FOU_ATTR_PORT]);
                cfg->udp_config.local_udp_port = port;
        }

        if (info->attrs[FOU_ATTR_IPPROTO])
                cfg->protocol = nla_get_u8(info->attrs[FOU_ATTR_IPPROTO]);

        if (info->attrs[FOU_ATTR_TYPE])
                cfg->type = nla_get_u8(info->attrs[FOU_ATTR_TYPE]);

        if (info->attrs[FOU_ATTR_REMCSUM_NOPARTIAL])
                cfg->flags |= FOU_F_REMCSUM_NOPARTIAL;

        if (cfg->udp_config.family == AF_INET) {
                if (info->attrs[FOU_ATTR_LOCAL_V4]) {
                        attr = info->attrs[FOU_ATTR_LOCAL_V4];
                        cfg->udp_config.local_ip.s_addr = nla_get_in_addr(attr);
                        has_local = true;
                }

                if (info->attrs[FOU_ATTR_PEER_V4]) {
                        attr = info->attrs[FOU_ATTR_PEER_V4];
                        cfg->udp_config.peer_ip.s_addr = nla_get_in_addr(attr);
                        has_peer = true;
                }
#if IS_ENABLED(CONFIG_IPV6)
        } else {
                if (info->attrs[FOU_ATTR_LOCAL_V6]) {
                        attr = info->attrs[FOU_ATTR_LOCAL_V6];
                        cfg->udp_config.local_ip6 = nla_get_in6_addr(attr);
                        has_local = true;
                }

                if (info->attrs[FOU_ATTR_PEER_V6]) {
                        attr = info->attrs[FOU_ATTR_PEER_V6];
                        cfg->udp_config.peer_ip6 = nla_get_in6_addr(attr);
                        has_peer = true;
                }
#endif
        }

        if (has_peer) {
                if (info->attrs[FOU_ATTR_PEER_PORT]) {
                        port = nla_get_be16(info->attrs[FOU_ATTR_PEER_PORT]);
                        cfg->udp_config.peer_udp_port = port;
                } else {
                        return -EINVAL;
                }
        }

        if (info->attrs[FOU_ATTR_IFINDEX]) {
                if (!has_local)
                        return -EINVAL;

                ifindex = nla_get_s32(info->attrs[FOU_ATTR_IFINDEX]);

                cfg->udp_config.bind_ifindex = ifindex;
        }

        return 0;
}

int fou_nl_add_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct fou_cfg cfg;
        int err;

        err = parse_nl_config(info, &cfg);
        if (err)
                return err;

        return fou_create(net, &cfg, NULL);
}

int fou_nl_del_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct fou_cfg cfg;
        int err;

        err = parse_nl_config(info, &cfg);
        if (err)
                return err;

        return fou_destroy(net, &cfg);
}

static int fou_fill_info(struct fou *fou, struct sk_buff *msg)
{
        struct sock *sk = fou->sock->sk;

        if (nla_put_u8(msg, FOU_ATTR_AF, fou->sock->sk->sk_family) ||
            nla_put_be16(msg, FOU_ATTR_PORT, fou->port) ||
            nla_put_be16(msg, FOU_ATTR_PEER_PORT, sk->sk_dport) ||
            nla_put_u8(msg, FOU_ATTR_IPPROTO, fou->protocol) ||
            nla_put_u8(msg, FOU_ATTR_TYPE, fou->type) ||
            nla_put_s32(msg, FOU_ATTR_IFINDEX, sk->sk_bound_dev_if))
                return -1;

        if (fou->flags & FOU_F_REMCSUM_NOPARTIAL)
                if (nla_put_flag(msg, FOU_ATTR_REMCSUM_NOPARTIAL))
                        return -1;

        if (fou->sock->sk->sk_family == AF_INET) {
                if (nla_put_in_addr(msg, FOU_ATTR_LOCAL_V4, sk->sk_rcv_saddr))
                        return -1;

                if (nla_put_in_addr(msg, FOU_ATTR_PEER_V4, sk->sk_daddr))
                        return -1;
#if IS_ENABLED(CONFIG_IPV6)
        } else {
                if (nla_put_in6_addr(msg, FOU_ATTR_LOCAL_V6,
                                     &sk->sk_v6_rcv_saddr))
                        return -1;

                if (nla_put_in6_addr(msg, FOU_ATTR_PEER_V6, &sk->sk_v6_daddr))
                        return -1;
#endif
        }

        return 0;
}

static int fou_dump_info(struct fou *fou, u32 portid, u32 seq,
                         u32 flags, struct sk_buff *skb, u8 cmd)
{
        void *hdr;

        hdr = genlmsg_put(skb, portid, seq, &fou_nl_family, flags, cmd);
        if (!hdr)
                return -ENOMEM;

        if (fou_fill_info(fou, skb) < 0)
                goto nla_put_failure;

        genlmsg_end(skb, hdr);
        return 0;

nla_put_failure:
        genlmsg_cancel(skb, hdr);
        return -EMSGSIZE;
}

int fou_nl_get_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct fou_net *fn = net_generic(net, fou_net_id);
        struct sk_buff *msg;
        struct fou_cfg cfg;
        struct fou *fout;
        __be16 port;
        u8 family;
        int ret;

        ret = parse_nl_config(info, &cfg);
        if (ret)
                return ret;
        port = cfg.udp_config.local_udp_port;
        if (port == 0)
                return -EINVAL;

        family = cfg.udp_config.family;
        if (family != AF_INET && family != AF_INET6)
                return -EINVAL;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        ret = -ESRCH;
        mutex_lock(&fn->fou_lock);
        list_for_each_entry(fout, &fn->fou_list, list) {
                if (fou_cfg_cmp(fout, &cfg)) {
                        ret = fou_dump_info(fout, info->snd_portid,
                                            info->snd_seq, 0, msg,
                                            info->genlhdr->cmd);
                        break;
                }
        }
        mutex_unlock(&fn->fou_lock);
        if (ret < 0)
                goto out_free;

        return genlmsg_reply(msg, info);

out_free:
        nlmsg_free(msg);
        return ret;
}

int fou_nl_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct net *net = sock_net(skb->sk);
        struct fou_net *fn = net_generic(net, fou_net_id);
        struct fou *fout;
        int idx = 0, ret;

        mutex_lock(&fn->fou_lock);
        list_for_each_entry(fout, &fn->fou_list, list) {
                if (idx++ < cb->args[0])
                        continue;
                ret = fou_dump_info(fout, NETLINK_CB(cb->skb).portid,
                                    cb->nlh->nlmsg_seq, NLM_F_MULTI,
                                    skb, FOU_CMD_GET);
                if (ret)
                        break;
        }
        mutex_unlock(&fn->fou_lock);

        cb->args[0] = idx;
        return skb->len;
}

static struct genl_family fou_nl_family __ro_after_init = {
        .hdrsize        = 0,
        .name                = FOU_GENL_NAME,
        .version        = FOU_GENL_VERSION,
        .maxattr        = FOU_ATTR_MAX,
        .policy                = fou_nl_policy,
        .netnsok        = true,
        .module                = THIS_MODULE,
        .small_ops        = fou_nl_ops,
        .n_small_ops        = ARRAY_SIZE(fou_nl_ops),
        .resv_start_op        = FOU_CMD_GET + 1,
};

size_t fou_encap_hlen(struct ip_tunnel_encap *e)
{
        return sizeof(struct udphdr);
}
EXPORT_SYMBOL(fou_encap_hlen);

size_t gue_encap_hlen(struct ip_tunnel_encap *e)
{
        size_t len;
        bool need_priv = false;

        len = sizeof(struct udphdr) + sizeof(struct guehdr);

        if (e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) {
                len += GUE_PLEN_REMCSUM;
                need_priv = true;
        }

        len += need_priv ? GUE_LEN_PRIV : 0;

        return len;
}
EXPORT_SYMBOL(gue_encap_hlen);

int __fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
                       u8 *protocol, __be16 *sport, int type)
{
        int err;

        err = iptunnel_handle_offloads(skb, type);
        if (err)
                return err;

        *sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
                                                skb, 0, 0, false);

        return 0;
}
EXPORT_SYMBOL(__fou_build_header);

int __gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
                       u8 *protocol, __be16 *sport, int type)
{
        struct guehdr *guehdr;
        size_t hdrlen, optlen = 0;
        void *data;
        bool need_priv = false;
        int err;

        if ((e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) &&
            skb->ip_summed == CHECKSUM_PARTIAL) {
                optlen += GUE_PLEN_REMCSUM;
                type |= SKB_GSO_TUNNEL_REMCSUM;
                need_priv = true;
        }

        optlen += need_priv ? GUE_LEN_PRIV : 0;

        err = iptunnel_handle_offloads(skb, type);
        if (err)
                return err;

        /* Get source port (based on flow hash) before skb_push */
        *sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
                                                skb, 0, 0, false);

        hdrlen = sizeof(struct guehdr) + optlen;

        skb_push(skb, hdrlen);

        guehdr = (struct guehdr *)skb->data;

        guehdr->control = 0;
        guehdr->version = 0;
        guehdr->hlen = optlen >> 2;
        guehdr->flags = 0;
        guehdr->proto_ctype = *protocol;

        data = &guehdr[1];

        if (need_priv) {
                __be32 *flags = data;

                guehdr->flags |= GUE_FLAG_PRIV;
                *flags = 0;
                data += GUE_LEN_PRIV;

                if (type & SKB_GSO_TUNNEL_REMCSUM) {
                        u16 csum_start = skb_checksum_start_offset(skb);
                        __be16 *pd = data;

                        if (csum_start < hdrlen)
                                return -EINVAL;

                        csum_start -= hdrlen;
                        pd[0] = htons(csum_start);
                        pd[1] = htons(csum_start + skb->csum_offset);

                        if (!skb_is_gso(skb)) {
                                skb->ip_summed = CHECKSUM_NONE;
                                skb->encapsulation = 0;
                        }

                        *flags |= GUE_PFLAG_REMCSUM;
                        data += GUE_PLEN_REMCSUM;
                }

        }

        return 0;
}
EXPORT_SYMBOL(__gue_build_header);

#ifdef CONFIG_NET_FOU_IP_TUNNELS

static void fou_build_udp(struct sk_buff *skb, struct ip_tunnel_encap *e,
                          struct flowi4 *fl4, u8 *protocol, __be16 sport)
{
        struct udphdr *uh;

        skb_push(skb, sizeof(struct udphdr));
        skb_reset_transport_header(skb);

        uh = udp_hdr(skb);

        uh->dest = e->dport;
        uh->source = sport;
        uh->len = htons(skb->len);
        udp_set_csum(!(e->flags & TUNNEL_ENCAP_FLAG_CSUM), skb,
                     fl4->saddr, fl4->daddr, skb->len);

        *protocol = IPPROTO_UDP;
}

static int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
                            u8 *protocol, struct flowi4 *fl4)
{
        int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM :
                                                       SKB_GSO_UDP_TUNNEL;
        __be16 sport;
        int err;

        err = __fou_build_header(skb, e, protocol, &sport, type);
        if (err)
                return err;

        fou_build_udp(skb, e, fl4, protocol, sport);

        return 0;
}

static int gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
                            u8 *protocol, struct flowi4 *fl4)
{
        int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM :
                                                       SKB_GSO_UDP_TUNNEL;
        __be16 sport;
        int err;

        err = __gue_build_header(skb, e, protocol, &sport, type);
        if (err)
                return err;

        fou_build_udp(skb, e, fl4, protocol, sport);

        return 0;
}

static int gue_err_proto_handler(int proto, struct sk_buff *skb, u32 info)
{
        const struct net_protocol *ipprot = rcu_dereference(inet_protos[proto]);

        if (ipprot && ipprot->err_handler) {
                if (!ipprot->err_handler(skb, info))
                        return 0;
        }

        return -ENOENT;
}

static int gue_err(struct sk_buff *skb, u32 info)
{
        int transport_offset = skb_transport_offset(skb);
        struct guehdr *guehdr;
        size_t len, optlen;
        int ret;

        len = sizeof(struct udphdr) + sizeof(struct guehdr);
        if (!pskb_may_pull(skb, transport_offset + len))
                return -EINVAL;

        guehdr = (struct guehdr *)&udp_hdr(skb)[1];

        switch (guehdr->version) {
        case 0: /* Full GUE header present */
                break;
        case 1: {
                /* Direct encapsulation of IPv4 or IPv6 */
                skb_set_transport_header(skb, -(int)sizeof(struct icmphdr));

                switch (((struct iphdr *)guehdr)->version) {
                case 4:
                        ret = gue_err_proto_handler(IPPROTO_IPIP, skb, info);
                        goto out;
#if IS_ENABLED(CONFIG_IPV6)
                case 6:
                        ret = gue_err_proto_handler(IPPROTO_IPV6, skb, info);
                        goto out;
#endif
                default:
                        ret = -EOPNOTSUPP;
                        goto out;
                }
        }
        default: /* Undefined version */
                return -EOPNOTSUPP;
        }

        if (guehdr->control)
                return -ENOENT;

        optlen = guehdr->hlen << 2;

        if (!pskb_may_pull(skb, transport_offset + len + optlen))
                return -EINVAL;

        guehdr = (struct guehdr *)&udp_hdr(skb)[1];
        if (validate_gue_flags(guehdr, optlen))
                return -EINVAL;

        /* Handling exceptions for direct UDP encapsulation in GUE would lead to
         * recursion. Besides, this kind of encapsulation can't even be
         * configured currently. Discard this.
         */
        if (guehdr->proto_ctype == IPPROTO_UDP ||
            guehdr->proto_ctype == IPPROTO_UDPLITE)
                return -EOPNOTSUPP;

        skb_set_transport_header(skb, -(int)sizeof(struct icmphdr));
        ret = gue_err_proto_handler(guehdr->proto_ctype, skb, info);

out:
        skb_set_transport_header(skb, transport_offset);
        return ret;
}


static const struct ip_tunnel_encap_ops fou_iptun_ops = {
        .encap_hlen = fou_encap_hlen,
        .build_header = fou_build_header,
        .err_handler = gue_err,
};

static const struct ip_tunnel_encap_ops gue_iptun_ops = {
        .encap_hlen = gue_encap_hlen,
        .build_header = gue_build_header,
        .err_handler = gue_err,
};

static int ip_tunnel_encap_add_fou_ops(void)
{
        int ret;

        ret = ip_tunnel_encap_add_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
        if (ret < 0) {
                pr_err("can't add fou ops\n");
                return ret;
        }

        ret = ip_tunnel_encap_add_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
        if (ret < 0) {
                pr_err("can't add gue ops\n");
                ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
                return ret;
        }

        return 0;
}

static void ip_tunnel_encap_del_fou_ops(void)
{
        ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
        ip_tunnel_encap_del_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
}

#else

static int ip_tunnel_encap_add_fou_ops(void)
{
        return 0;
}

static void ip_tunnel_encap_del_fou_ops(void)
{
}

#endif

static __net_init int fou_init_net(struct net *net)
{
        struct fou_net *fn = net_generic(net, fou_net_id);

        INIT_LIST_HEAD(&fn->fou_list);
        mutex_init(&fn->fou_lock);
        return 0;
}

static __net_exit void fou_exit_net(struct net *net)
{
        struct fou_net *fn = net_generic(net, fou_net_id);
        struct fou *fou, *next;

        /* Close all the FOU sockets */
        mutex_lock(&fn->fou_lock);
        list_for_each_entry_safe(fou, next, &fn->fou_list, list)
                fou_release(fou);
        mutex_unlock(&fn->fou_lock);
}

static struct pernet_operations fou_net_ops = {
        .init = fou_init_net,
        .exit = fou_exit_net,
        .id   = &fou_net_id,
        .size = sizeof(struct fou_net),
};

static int __init fou_init(void)
{
        int ret;

        ret = register_pernet_device(&fou_net_ops);
        if (ret)
                goto exit;

        ret = genl_register_family(&fou_nl_family);
        if (ret < 0)
                goto unregister;

        ret = register_fou_bpf();
        if (ret < 0)
                goto kfunc_failed;

        ret = ip_tunnel_encap_add_fou_ops();
        if (ret == 0)
                return 0;

kfunc_failed:
        genl_unregister_family(&fou_nl_family);
unregister:
        unregister_pernet_device(&fou_net_ops);
exit:
        return ret;
}

static void __exit fou_fini(void)
{
        ip_tunnel_encap_del_fou_ops();
        genl_unregister_family(&fou_nl_family);
        unregister_pernet_device(&fou_net_ops);
}

module_init(fou_init);
module_exit(fou_fini);
MODULE_AUTHOR("Tom Herbert <therbert@google.com>");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Foo over UDP");































































































































































































   67 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
// SPDX-License-Identifier: GPL-2.0
/* RTT/RTO calculation.
 *
 * Adapted from TCP for AF_RXRPC by David Howells (dhowells@redhat.com)
 *
 * https://tools.ietf.org/html/rfc6298
 * https://tools.ietf.org/html/rfc1122#section-4.2.3.1
 * http://ccr.sigcomm.org/archive/1995/jan95/ccr-9501-partridge87.pdf
 */

#include <linux/net.h>
#include "ar-internal.h"

#define RXRPC_RTO_MAX        (120 * USEC_PER_SEC)
#define RXRPC_TIMEOUT_INIT ((unsigned int)(1 * MSEC_PER_SEC)) /* RFC6298 2.1 initial RTO value */
#define rxrpc_jiffies32 ((u32)jiffies)                /* As rxrpc_jiffies32 */

static u32 rxrpc_rto_min_us(struct rxrpc_peer *peer)
{
        return 200;
}

static u32 __rxrpc_set_rto(const struct rxrpc_peer *peer)
{
        return (peer->srtt_us >> 3) + peer->rttvar_us;
}

static u32 rxrpc_bound_rto(u32 rto)
{
        return min(rto, RXRPC_RTO_MAX);
}

/*
 * Called to compute a smoothed rtt estimate. The data fed to this
 * routine either comes from timestamps, or from segments that were
 * known _not_ to have been retransmitted [see Karn/Partridge
 * Proceedings SIGCOMM 87]. The algorithm is from the SIGCOMM 88
 * piece by Van Jacobson.
 * NOTE: the next three routines used to be one big routine.
 * To save cycles in the RFC 1323 implementation it was better to break
 * it up into three procedures. -- erics
 */
static void rxrpc_rtt_estimator(struct rxrpc_peer *peer, long sample_rtt_us)
{
        long m = sample_rtt_us; /* RTT */
        u32 srtt = peer->srtt_us;

        /*        The following amusing code comes from Jacobson's
         *        article in SIGCOMM '88.  Note that rtt and mdev
         *        are scaled versions of rtt and mean deviation.
         *        This is designed to be as fast as possible
         *        m stands for "measurement".
         *
         *        On a 1990 paper the rto value is changed to:
         *        RTO = rtt + 4 * mdev
         *
         * Funny. This algorithm seems to be very broken.
         * These formulae increase RTO, when it should be decreased, increase
         * too slowly, when it should be increased quickly, decrease too quickly
         * etc. I guess in BSD RTO takes ONE value, so that it is absolutely
         * does not matter how to _calculate_ it. Seems, it was trap
         * that VJ failed to avoid. 8)
         */
        if (srtt != 0) {
                m -= (srtt >> 3);        /* m is now error in rtt est */
                srtt += m;                /* rtt = 7/8 rtt + 1/8 new */
                if (m < 0) {
                        m = -m;                /* m is now abs(error) */
                        m -= (peer->mdev_us >> 2);   /* similar update on mdev */
                        /* This is similar to one of Eifel findings.
                         * Eifel blocks mdev updates when rtt decreases.
                         * This solution is a bit different: we use finer gain
                         * for mdev in this case (alpha*beta).
                         * Like Eifel it also prevents growth of rto,
                         * but also it limits too fast rto decreases,
                         * happening in pure Eifel.
                         */
                        if (m > 0)
                                m >>= 3;
                } else {
                        m -= (peer->mdev_us >> 2);   /* similar update on mdev */
                }

                peer->mdev_us += m;                /* mdev = 3/4 mdev + 1/4 new */
                if (peer->mdev_us > peer->mdev_max_us) {
                        peer->mdev_max_us = peer->mdev_us;
                        if (peer->mdev_max_us > peer->rttvar_us)
                                peer->rttvar_us = peer->mdev_max_us;
                }
        } else {
                /* no previous measure. */
                srtt = m << 3;                /* take the measured time to be rtt */
                peer->mdev_us = m << 1;        /* make sure rto = 3*rtt */
                peer->rttvar_us = max(peer->mdev_us, rxrpc_rto_min_us(peer));
                peer->mdev_max_us = peer->rttvar_us;
        }

        peer->srtt_us = max(1U, srtt);
}

/*
 * Calculate rto without backoff.  This is the second half of Van Jacobson's
 * routine referred to above.
 */
static void rxrpc_set_rto(struct rxrpc_peer *peer)
{
        u32 rto;

        /* 1. If rtt variance happened to be less 50msec, it is hallucination.
         *    It cannot be less due to utterly erratic ACK generation made
         *    at least by solaris and freebsd. "Erratic ACKs" has _nothing_
         *    to do with delayed acks, because at cwnd>2 true delack timeout
         *    is invisible. Actually, Linux-2.4 also generates erratic
         *    ACKs in some circumstances.
         */
        rto = __rxrpc_set_rto(peer);

        /* 2. Fixups made earlier cannot be right.
         *    If we do not estimate RTO correctly without them,
         *    all the algo is pure shit and should be replaced
         *    with correct one. It is exactly, which we pretend to do.
         */

        /* NOTE: clamping at RXRPC_RTO_MIN is not required, current algo
         * guarantees that rto is higher.
         */
        peer->rto_us = rxrpc_bound_rto(rto);
}

static void rxrpc_ack_update_rtt(struct rxrpc_peer *peer, long rtt_us)
{
        if (rtt_us < 0)
                return;

        //rxrpc_update_rtt_min(peer, rtt_us);
        rxrpc_rtt_estimator(peer, rtt_us);
        rxrpc_set_rto(peer);

        /* RFC6298: only reset backoff on valid RTT measurement. */
        peer->backoff = 0;
}

/*
 * Add RTT information to cache.  This is called in softirq mode and has
 * exclusive access to the peer RTT data.
 */
void rxrpc_peer_add_rtt(struct rxrpc_call *call, enum rxrpc_rtt_rx_trace why,
                        int rtt_slot,
                        rxrpc_serial_t send_serial, rxrpc_serial_t resp_serial,
                        ktime_t send_time, ktime_t resp_time)
{
        struct rxrpc_peer *peer = call->peer;
        s64 rtt_us;

        rtt_us = ktime_to_us(ktime_sub(resp_time, send_time));
        if (rtt_us < 0)
                return;

        spin_lock(&peer->rtt_input_lock);
        rxrpc_ack_update_rtt(peer, rtt_us);
        if (peer->rtt_count < 3)
                peer->rtt_count++;
        spin_unlock(&peer->rtt_input_lock);

        trace_rxrpc_rtt_rx(call, why, rtt_slot, send_serial, resp_serial,
                           peer->srtt_us >> 3, peer->rto_us);
}

/*
 * Get the retransmission timeout to set in nanoseconds, backing it off each
 * time we retransmit.
 */
ktime_t rxrpc_get_rto_backoff(struct rxrpc_peer *peer, bool retrans)
{
        u64 timo_us;
        u32 backoff = READ_ONCE(peer->backoff);

        timo_us = peer->rto_us;
        timo_us <<= backoff;
        if (retrans && timo_us * 2 <= RXRPC_RTO_MAX)
                WRITE_ONCE(peer->backoff, backoff + 1);

        if (timo_us < 1)
                timo_us = 1;

        return ns_to_ktime(timo_us * NSEC_PER_USEC);
}

void rxrpc_peer_init_rtt(struct rxrpc_peer *peer)
{
        peer->rto_us        = RXRPC_TIMEOUT_INIT;
        peer->mdev_us        = RXRPC_TIMEOUT_INIT;
        peer->backoff        = 0;
        //minmax_reset(&peer->rtt_min, rxrpc_jiffies32, ~0U);
}































































































































    4 




    4 


































    6 


    6 
    6 



    2 







    2 
    2 

    6 
















































    2 
    3 













    3 











































    2 

    2 




    2 




























    2 






    2 





    2 
    2 
    1 


    1 





    2 

















    2 


    1 

    1 







    2 



    2 





    2 
    2 













    2 









































































































































































    2 

    2 
    2 

















    2 

    2 






























    2 




    2 




    2 








    2 
    2 






























    2 





    2 
    2 






































    2 
    2 







    1 










    1 












    1 

    1 






























    6 






































    6 




    6 

































































    1 
    1 















    1 














    5 




    1 









    3 
    2 





    1 

    1 
    1 





    1 







    1 





    1 

    5 






















    6 


    6 





    3 



    3 





    3 


    3 


    3 





    3 

    3 
    3 
    2 




    2 



    2 






    2 

    6 















    6 


































































    6 






    5 










    2 















    1 



































    2 



    1 
    6 





















































































    1 


    1 
































































































































    3 



    4 
    4 


    4 



    2 

    2 
    2 
























   25 











    4 

























































   26 







































































































































   26 

   26 

    4 
































   26 


   26 


   25 







































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) B.A.T.M.A.N. contributors:
 *
 * Antonio Quartulli
 */

#include "distributed-arp-table.h"
#include "main.h"

#include <linux/unaligned.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/byteorder/generic.h>
#include <linux/container_of.h>
#include <linux/errno.h>
#include <linux/etherdevice.h>
#include <linux/gfp.h>
#include <linux/if_arp.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/jiffies.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/netlink.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/stddef.h>
#include <linux/string.h>
#include <linux/udp.h>
#include <linux/workqueue.h>
#include <net/arp.h>
#include <net/genetlink.h>
#include <net/netlink.h>
#include <net/sock.h>
#include <uapi/linux/batman_adv.h>

#include "bridge_loop_avoidance.h"
#include "hard-interface.h"
#include "hash.h"
#include "log.h"
#include "netlink.h"
#include "originator.h"
#include "send.h"
#include "soft-interface.h"
#include "translation-table.h"
#include "tvlv.h"

enum batadv_bootpop {
        BATADV_BOOTREPLY        = 2,
};

enum batadv_boothtype {
        BATADV_HTYPE_ETHERNET        = 1,
};

enum batadv_dhcpoptioncode {
        BATADV_DHCP_OPT_PAD                = 0,
        BATADV_DHCP_OPT_MSG_TYPE        = 53,
        BATADV_DHCP_OPT_END                = 255,
};

enum batadv_dhcptype {
        BATADV_DHCPACK                = 5,
};

/* { 99, 130, 83, 99 } */
#define BATADV_DHCP_MAGIC 1669485411

struct batadv_dhcp_packet {
        __u8 op;
        __u8 htype;
        __u8 hlen;
        __u8 hops;
        __be32 xid;
        __be16 secs;
        __be16 flags;
        __be32 ciaddr;
        __be32 yiaddr;
        __be32 siaddr;
        __be32 giaddr;
        __u8 chaddr[16];
        __u8 sname[64];
        __u8 file[128];
        __be32 magic;
        /* __u8 options[]; */
};

#define BATADV_DHCP_YIADDR_LEN sizeof(((struct batadv_dhcp_packet *)0)->yiaddr)
#define BATADV_DHCP_CHADDR_LEN sizeof(((struct batadv_dhcp_packet *)0)->chaddr)

static void batadv_dat_purge(struct work_struct *work);

/**
 * batadv_dat_start_timer() - initialise the DAT periodic worker
 * @bat_priv: the bat priv with all the soft interface information
 */
static void batadv_dat_start_timer(struct batadv_priv *bat_priv)
{
        queue_delayed_work(batadv_event_workqueue, &bat_priv->dat.work,
                           msecs_to_jiffies(10000));
}

/**
 * batadv_dat_entry_release() - release dat_entry from lists and queue for free
 *  after rcu grace period
 * @ref: kref pointer of the dat_entry
 */
static void batadv_dat_entry_release(struct kref *ref)
{
        struct batadv_dat_entry *dat_entry;

        dat_entry = container_of(ref, struct batadv_dat_entry, refcount);

        kfree_rcu(dat_entry, rcu);
}

/**
 * batadv_dat_entry_put() - decrement the dat_entry refcounter and possibly
 *  release it
 * @dat_entry: dat_entry to be free'd
 */
static void batadv_dat_entry_put(struct batadv_dat_entry *dat_entry)
{
        if (!dat_entry)
                return;

        kref_put(&dat_entry->refcount, batadv_dat_entry_release);
}

/**
 * batadv_dat_to_purge() - check whether a dat_entry has to be purged or not
 * @dat_entry: the entry to check
 *
 * Return: true if the entry has to be purged now, false otherwise.
 */
static bool batadv_dat_to_purge(struct batadv_dat_entry *dat_entry)
{
        return batadv_has_timed_out(dat_entry->last_update,
                                    BATADV_DAT_ENTRY_TIMEOUT);
}

/**
 * __batadv_dat_purge() - delete entries from the DAT local storage
 * @bat_priv: the bat priv with all the soft interface information
 * @to_purge: function in charge to decide whether an entry has to be purged or
 *              not. This function takes the dat_entry as argument and has to
 *              returns a boolean value: true is the entry has to be deleted,
 *              false otherwise
 *
 * Loops over each entry in the DAT local storage and deletes it if and only if
 * the to_purge function passed as argument returns true.
 */
static void __batadv_dat_purge(struct batadv_priv *bat_priv,
                               bool (*to_purge)(struct batadv_dat_entry *))
{
        spinlock_t *list_lock; /* protects write access to the hash lists */
        struct batadv_dat_entry *dat_entry;
        struct hlist_node *node_tmp;
        struct hlist_head *head;
        u32 i;

        if (!bat_priv->dat.hash)
                return;

        for (i = 0; i < bat_priv->dat.hash->size; i++) {
                head = &bat_priv->dat.hash->table[i];
                list_lock = &bat_priv->dat.hash->list_locks[i];

                spin_lock_bh(list_lock);
                hlist_for_each_entry_safe(dat_entry, node_tmp, head,
                                          hash_entry) {
                        /* if a helper function has been passed as parameter,
                         * ask it if the entry has to be purged or not
                         */
                        if (to_purge && !to_purge(dat_entry))
                                continue;

                        hlist_del_rcu(&dat_entry->hash_entry);
                        batadv_dat_entry_put(dat_entry);
                }
                spin_unlock_bh(list_lock);
        }
}

/**
 * batadv_dat_purge() - periodic task that deletes old entries from the local
 *  DAT hash table
 * @work: kernel work struct
 */
static void batadv_dat_purge(struct work_struct *work)
{
        struct delayed_work *delayed_work;
        struct batadv_priv_dat *priv_dat;
        struct batadv_priv *bat_priv;

        delayed_work = to_delayed_work(work);
        priv_dat = container_of(delayed_work, struct batadv_priv_dat, work);
        bat_priv = container_of(priv_dat, struct batadv_priv, dat);

        __batadv_dat_purge(bat_priv, batadv_dat_to_purge);
        batadv_dat_start_timer(bat_priv);
}

/**
 * batadv_compare_dat() - comparing function used in the local DAT hash table
 * @node: node in the local table
 * @data2: second object to compare the node to
 *
 * Return: true if the two entries are the same, false otherwise.
 */
static bool batadv_compare_dat(const struct hlist_node *node, const void *data2)
{
        const void *data1 = container_of(node, struct batadv_dat_entry,
                                         hash_entry);

        return memcmp(data1, data2, sizeof(__be32)) == 0;
}

/**
 * batadv_arp_hw_src() - extract the hw_src field from an ARP packet
 * @skb: ARP packet
 * @hdr_size: size of the possible header before the ARP packet
 *
 * Return: the value of the hw_src field in the ARP packet.
 */
static u8 *batadv_arp_hw_src(struct sk_buff *skb, int hdr_size)
{
        u8 *addr;

        addr = (u8 *)(skb->data + hdr_size);
        addr += ETH_HLEN + sizeof(struct arphdr);

        return addr;
}

/**
 * batadv_arp_ip_src() - extract the ip_src field from an ARP packet
 * @skb: ARP packet
 * @hdr_size: size of the possible header before the ARP packet
 *
 * Return: the value of the ip_src field in the ARP packet.
 */
static __be32 batadv_arp_ip_src(struct sk_buff *skb, int hdr_size)
{
        return *(__force __be32 *)(batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN);
}

/**
 * batadv_arp_hw_dst() - extract the hw_dst field from an ARP packet
 * @skb: ARP packet
 * @hdr_size: size of the possible header before the ARP packet
 *
 * Return: the value of the hw_dst field in the ARP packet.
 */
static u8 *batadv_arp_hw_dst(struct sk_buff *skb, int hdr_size)
{
        return batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN + 4;
}

/**
 * batadv_arp_ip_dst() - extract the ip_dst field from an ARP packet
 * @skb: ARP packet
 * @hdr_size: size of the possible header before the ARP packet
 *
 * Return: the value of the ip_dst field in the ARP packet.
 */
static __be32 batadv_arp_ip_dst(struct sk_buff *skb, int hdr_size)
{
        u8 *dst = batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN * 2 + 4;

        return *(__force __be32 *)dst;
}

/**
 * batadv_hash_dat() - compute the hash value for an IP address
 * @data: data to hash
 * @size: size of the hash table
 *
 * Return: the selected index in the hash table for the given data.
 */
static u32 batadv_hash_dat(const void *data, u32 size)
{
        u32 hash = 0;
        const struct batadv_dat_entry *dat = data;
        const unsigned char *key;
        __be16 vid;
        u32 i;

        key = (__force const unsigned char *)&dat->ip;
        for (i = 0; i < sizeof(dat->ip); i++) {
                hash += key[i];
                hash += (hash << 10);
                hash ^= (hash >> 6);
        }

        vid = htons(dat->vid);
        key = (__force const unsigned char *)&vid;
        for (i = 0; i < sizeof(dat->vid); i++) {
                hash += key[i];
                hash += (hash << 10);
                hash ^= (hash >> 6);
        }

        hash += (hash << 3);
        hash ^= (hash >> 11);
        hash += (hash << 15);

        return hash % size;
}

/**
 * batadv_dat_entry_hash_find() - look for a given dat_entry in the local hash
 * table
 * @bat_priv: the bat priv with all the soft interface information
 * @ip: search key
 * @vid: VLAN identifier
 *
 * Return: the dat_entry if found, NULL otherwise.
 */
static struct batadv_dat_entry *
batadv_dat_entry_hash_find(struct batadv_priv *bat_priv, __be32 ip,
                           unsigned short vid)
{
        struct hlist_head *head;
        struct batadv_dat_entry to_find, *dat_entry, *dat_entry_tmp = NULL;
        struct batadv_hashtable *hash = bat_priv->dat.hash;
        u32 index;

        if (!hash)
                return NULL;

        to_find.ip = ip;
        to_find.vid = vid;

        index = batadv_hash_dat(&to_find, hash->size);
        head = &hash->table[index];

        rcu_read_lock();
        hlist_for_each_entry_rcu(dat_entry, head, hash_entry) {
                if (dat_entry->ip != ip)
                        continue;

                if (!kref_get_unless_zero(&dat_entry->refcount))
                        continue;

                dat_entry_tmp = dat_entry;
                break;
        }
        rcu_read_unlock();

        return dat_entry_tmp;
}

/**
 * batadv_dat_entry_add() - add a new dat entry or update it if already exists
 * @bat_priv: the bat priv with all the soft interface information
 * @ip: ipv4 to add/edit
 * @mac_addr: mac address to assign to the given ipv4
 * @vid: VLAN identifier
 */
static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip,
                                 u8 *mac_addr, unsigned short vid)
{
        struct batadv_dat_entry *dat_entry;
        int hash_added;

        dat_entry = batadv_dat_entry_hash_find(bat_priv, ip, vid);
        /* if this entry is already known, just update it */
        if (dat_entry) {
                if (!batadv_compare_eth(dat_entry->mac_addr, mac_addr))
                        ether_addr_copy(dat_entry->mac_addr, mac_addr);
                dat_entry->last_update = jiffies;
                batadv_dbg(BATADV_DBG_DAT, bat_priv,
                           "Entry updated: %pI4 %pM (vid: %d)\n",
                           &dat_entry->ip, dat_entry->mac_addr,
                           batadv_print_vid(vid));
                goto out;
        }

        dat_entry = kmalloc(sizeof(*dat_entry), GFP_ATOMIC);
        if (!dat_entry)
                goto out;

        dat_entry->ip = ip;
        dat_entry->vid = vid;
        ether_addr_copy(dat_entry->mac_addr, mac_addr);
        dat_entry->last_update = jiffies;
        kref_init(&dat_entry->refcount);

        kref_get(&dat_entry->refcount);
        hash_added = batadv_hash_add(bat_priv->dat.hash, batadv_compare_dat,
                                     batadv_hash_dat, dat_entry,
                                     &dat_entry->hash_entry);

        if (unlikely(hash_added != 0)) {
                /* remove the reference for the hash */
                batadv_dat_entry_put(dat_entry);
                goto out;
        }

        batadv_dbg(BATADV_DBG_DAT, bat_priv, "New entry added: %pI4 %pM (vid: %d)\n",
                   &dat_entry->ip, dat_entry->mac_addr, batadv_print_vid(vid));

out:
        batadv_dat_entry_put(dat_entry);
}

#ifdef CONFIG_BATMAN_ADV_DEBUG

/**
 * batadv_dbg_arp() - print a debug message containing all the ARP packet
 *  details
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: ARP packet
 * @hdr_size: size of the possible header before the ARP packet
 * @msg: message to print together with the debugging information
 */
static void batadv_dbg_arp(struct batadv_priv *bat_priv, struct sk_buff *skb,
                           int hdr_size, char *msg)
{
        struct batadv_unicast_4addr_packet *unicast_4addr_packet;
        struct batadv_bcast_packet *bcast_pkt;
        u8 *orig_addr;
        __be32 ip_src, ip_dst;

        if (msg)
                batadv_dbg(BATADV_DBG_DAT, bat_priv, "%s\n", msg);

        ip_src = batadv_arp_ip_src(skb, hdr_size);
        ip_dst = batadv_arp_ip_dst(skb, hdr_size);
        batadv_dbg(BATADV_DBG_DAT, bat_priv,
                   "ARP MSG = [src: %pM-%pI4 dst: %pM-%pI4]\n",
                   batadv_arp_hw_src(skb, hdr_size), &ip_src,
                   batadv_arp_hw_dst(skb, hdr_size), &ip_dst);

        if (hdr_size < sizeof(struct batadv_unicast_packet))
                return;

        unicast_4addr_packet = (struct batadv_unicast_4addr_packet *)skb->data;

        switch (unicast_4addr_packet->u.packet_type) {
        case BATADV_UNICAST:
                batadv_dbg(BATADV_DBG_DAT, bat_priv,
                           "* encapsulated within a UNICAST packet\n");
                break;
        case BATADV_UNICAST_4ADDR:
                batadv_dbg(BATADV_DBG_DAT, bat_priv,
                           "* encapsulated within a UNICAST_4ADDR packet (src: %pM)\n",
                           unicast_4addr_packet->src);
                switch (unicast_4addr_packet->subtype) {
                case BATADV_P_DAT_DHT_PUT:
                        batadv_dbg(BATADV_DBG_DAT, bat_priv, "* type: DAT_DHT_PUT\n");
                        break;
                case BATADV_P_DAT_DHT_GET:
                        batadv_dbg(BATADV_DBG_DAT, bat_priv, "* type: DAT_DHT_GET\n");
                        break;
                case BATADV_P_DAT_CACHE_REPLY:
                        batadv_dbg(BATADV_DBG_DAT, bat_priv,
                                   "* type: DAT_CACHE_REPLY\n");
                        break;
                case BATADV_P_DATA:
                        batadv_dbg(BATADV_DBG_DAT, bat_priv, "* type: DATA\n");
                        break;
                default:
                        batadv_dbg(BATADV_DBG_DAT, bat_priv, "* type: Unknown (%u)!\n",
                                   unicast_4addr_packet->u.packet_type);
                }
                break;
        case BATADV_BCAST:
                bcast_pkt = (struct batadv_bcast_packet *)unicast_4addr_packet;
                orig_addr = bcast_pkt->orig;
                batadv_dbg(BATADV_DBG_DAT, bat_priv,
                           "* encapsulated within a BCAST packet (src: %pM)\n",
                           orig_addr);
                break;
        default:
                batadv_dbg(BATADV_DBG_DAT, bat_priv,
                           "* encapsulated within an unknown packet type (0x%x)\n",
                           unicast_4addr_packet->u.packet_type);
        }
}

#else

static void batadv_dbg_arp(struct batadv_priv *bat_priv, struct sk_buff *skb,
                           int hdr_size, char *msg)
{
}

#endif /* CONFIG_BATMAN_ADV_DEBUG */

/**
 * batadv_is_orig_node_eligible() - check whether a node can be a DHT candidate
 * @res: the array with the already selected candidates
 * @select: number of already selected candidates
 * @tmp_max: address of the currently evaluated node
 * @max: current round max address
 * @last_max: address of the last selected candidate
 * @candidate: orig_node under evaluation
 * @max_orig_node: last selected candidate
 *
 * Return: true if the node has been elected as next candidate or false
 * otherwise.
 */
static bool batadv_is_orig_node_eligible(struct batadv_dat_candidate *res,
                                         int select, batadv_dat_addr_t tmp_max,
                                         batadv_dat_addr_t max,
                                         batadv_dat_addr_t last_max,
                                         struct batadv_orig_node *candidate,
                                         struct batadv_orig_node *max_orig_node)
{
        bool ret = false;
        int j;

        /* check if orig node candidate is running DAT */
        if (!test_bit(BATADV_ORIG_CAPA_HAS_DAT, &candidate->capabilities))
                goto out;

        /* Check if this node has already been selected... */
        for (j = 0; j < select; j++)
                if (res[j].orig_node == candidate)
                        break;
        /* ..and possibly skip it */
        if (j < select)
                goto out;
        /* sanity check: has it already been selected? This should not happen */
        if (tmp_max > last_max)
                goto out;
        /* check if during this iteration an originator with a closer dht
         * address has already been found
         */
        if (tmp_max < max)
                goto out;
        /* this is an hash collision with the temporary selected node. Choose
         * the one with the lowest address
         */
        if (tmp_max == max && max_orig_node &&
            batadv_compare_eth(candidate->orig, max_orig_node->orig))
                goto out;

        ret = true;
out:
        return ret;
}

/**
 * batadv_choose_next_candidate() - select the next DHT candidate
 * @bat_priv: the bat priv with all the soft interface information
 * @cands: candidates array
 * @select: number of candidates already present in the array
 * @ip_key: key to look up in the DHT
 * @last_max: pointer where the address of the selected candidate will be saved
 */
static void batadv_choose_next_candidate(struct batadv_priv *bat_priv,
                                         struct batadv_dat_candidate *cands,
                                         int select, batadv_dat_addr_t ip_key,
                                         batadv_dat_addr_t *last_max)
{
        batadv_dat_addr_t max = 0;
        batadv_dat_addr_t tmp_max = 0;
        struct batadv_orig_node *orig_node, *max_orig_node = NULL;
        struct batadv_hashtable *hash = bat_priv->orig_hash;
        struct hlist_head *head;
        int i;

        /* if no node is eligible as candidate, leave the candidate type as
         * NOT_FOUND
         */
        cands[select].type = BATADV_DAT_CANDIDATE_NOT_FOUND;

        /* iterate over the originator list and find the node with the closest
         * dat_address which has not been selected yet
         */
        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];

                rcu_read_lock();
                hlist_for_each_entry_rcu(orig_node, head, hash_entry) {
                        /* the dht space is a ring using unsigned addresses */
                        tmp_max = BATADV_DAT_ADDR_MAX - orig_node->dat_addr +
                                  ip_key;

                        if (!batadv_is_orig_node_eligible(cands, select,
                                                          tmp_max, max,
                                                          *last_max, orig_node,
                                                          max_orig_node))
                                continue;

                        if (!kref_get_unless_zero(&orig_node->refcount))
                                continue;

                        max = tmp_max;
                        batadv_orig_node_put(max_orig_node);
                        max_orig_node = orig_node;
                }
                rcu_read_unlock();
        }
        if (max_orig_node) {
                cands[select].type = BATADV_DAT_CANDIDATE_ORIG;
                cands[select].orig_node = max_orig_node;
                batadv_dbg(BATADV_DBG_DAT, bat_priv,
                           "dat_select_candidates() %d: selected %pM addr=%u dist=%u\n",
                           select, max_orig_node->orig, max_orig_node->dat_addr,
                           max);
        }
        *last_max = max;
}

/**
 * batadv_dat_select_candidates() - select the nodes which the DHT message has
 *  to be sent to
 * @bat_priv: the bat priv with all the soft interface information
 * @ip_dst: ipv4 to look up in the DHT
 * @vid: VLAN identifier
 *
 * An originator O is selected if and only if its DHT_ID value is one of three
 * closest values (from the LEFT, with wrap around if needed) then the hash
 * value of the key. ip_dst is the key.
 *
 * Return: the candidate array of size BATADV_DAT_CANDIDATE_NUM.
 */
static struct batadv_dat_candidate *
batadv_dat_select_candidates(struct batadv_priv *bat_priv, __be32 ip_dst,
                             unsigned short vid)
{
        int select;
        batadv_dat_addr_t last_max = BATADV_DAT_ADDR_MAX, ip_key;
        struct batadv_dat_candidate *res;
        struct batadv_dat_entry dat;

        if (!bat_priv->orig_hash)
                return NULL;

        res = kmalloc_array(BATADV_DAT_CANDIDATES_NUM, sizeof(*res),
                            GFP_ATOMIC);
        if (!res)
                return NULL;

        dat.ip = ip_dst;
        dat.vid = vid;
        ip_key = (batadv_dat_addr_t)batadv_hash_dat(&dat,
                                                    BATADV_DAT_ADDR_MAX);

        batadv_dbg(BATADV_DBG_DAT, bat_priv,
                   "%s(): IP=%pI4 hash(IP)=%u\n", __func__, &ip_dst,
                   ip_key);

        for (select = 0; select < BATADV_DAT_CANDIDATES_NUM; select++)
                batadv_choose_next_candidate(bat_priv, res, select, ip_key,
                                             &last_max);

        return res;
}

/**
 * batadv_dat_forward_data() - copy and send payload to the selected candidates
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: payload to send
 * @ip: the DHT key
 * @vid: VLAN identifier
 * @packet_subtype: unicast4addr packet subtype to use
 *
 * This function copies the skb with pskb_copy() and is sent as a unicast packet
 * to each of the selected candidates.
 *
 * Return: true if the packet is sent to at least one candidate, false
 * otherwise.
 */
static bool batadv_dat_forward_data(struct batadv_priv *bat_priv,
                                    struct sk_buff *skb, __be32 ip,
                                    unsigned short vid, int packet_subtype)
{
        int i;
        bool ret = false;
        int send_status;
        struct batadv_neigh_node *neigh_node = NULL;
        struct sk_buff *tmp_skb;
        struct batadv_dat_candidate *cand;

        cand = batadv_dat_select_candidates(bat_priv, ip, vid);
        if (!cand)
                return ret;

        batadv_dbg(BATADV_DBG_DAT, bat_priv, "DHT_SEND for %pI4\n", &ip);

        for (i = 0; i < BATADV_DAT_CANDIDATES_NUM; i++) {
                if (cand[i].type == BATADV_DAT_CANDIDATE_NOT_FOUND)
                        continue;

                neigh_node = batadv_orig_router_get(cand[i].orig_node,
                                                    BATADV_IF_DEFAULT);
                if (!neigh_node)
                        goto free_orig;

                tmp_skb = pskb_copy_for_clone(skb, GFP_ATOMIC);
                if (!batadv_send_skb_prepare_unicast_4addr(bat_priv, tmp_skb,
                                                           cand[i].orig_node,
                                                           packet_subtype)) {
                        kfree_skb(tmp_skb);
                        goto free_neigh;
                }

                send_status = batadv_send_unicast_skb(tmp_skb, neigh_node);
                if (send_status == NET_XMIT_SUCCESS) {
                        /* count the sent packet */
                        switch (packet_subtype) {
                        case BATADV_P_DAT_DHT_GET:
                                batadv_inc_counter(bat_priv,
                                                   BATADV_CNT_DAT_GET_TX);
                                break;
                        case BATADV_P_DAT_DHT_PUT:
                                batadv_inc_counter(bat_priv,
                                                   BATADV_CNT_DAT_PUT_TX);
                                break;
                        }

                        /* packet sent to a candidate: return true */
                        ret = true;
                }
free_neigh:
                batadv_neigh_node_put(neigh_node);
free_orig:
                batadv_orig_node_put(cand[i].orig_node);
        }

        kfree(cand);
        return ret;
}

/**
 * batadv_dat_tvlv_container_update() - update the dat tvlv container after dat
 *  setting change
 * @bat_priv: the bat priv with all the soft interface information
 */
static void batadv_dat_tvlv_container_update(struct batadv_priv *bat_priv)
{
        char dat_mode;

        dat_mode = atomic_read(&bat_priv->distributed_arp_table);

        switch (dat_mode) {
        case 0:
                batadv_tvlv_container_unregister(bat_priv, BATADV_TVLV_DAT, 1);
                break;
        case 1:
                batadv_tvlv_container_register(bat_priv, BATADV_TVLV_DAT, 1,
                                               NULL, 0);
                break;
        }
}

/**
 * batadv_dat_status_update() - update the dat tvlv container after dat
 *  setting change
 * @net_dev: the soft interface net device
 */
void batadv_dat_status_update(struct net_device *net_dev)
{
        struct batadv_priv *bat_priv = netdev_priv(net_dev);

        batadv_dat_tvlv_container_update(bat_priv);
}

/**
 * batadv_dat_tvlv_ogm_handler_v1() - process incoming dat tvlv container
 * @bat_priv: the bat priv with all the soft interface information
 * @orig: the orig_node of the ogm
 * @flags: flags indicating the tvlv state (see batadv_tvlv_handler_flags)
 * @tvlv_value: tvlv buffer containing the gateway data
 * @tvlv_value_len: tvlv buffer length
 */
static void batadv_dat_tvlv_ogm_handler_v1(struct batadv_priv *bat_priv,
                                           struct batadv_orig_node *orig,
                                           u8 flags,
                                           void *tvlv_value, u16 tvlv_value_len)
{
        if (flags & BATADV_TVLV_HANDLER_OGM_CIFNOTFND)
                clear_bit(BATADV_ORIG_CAPA_HAS_DAT, &orig->capabilities);
        else
                set_bit(BATADV_ORIG_CAPA_HAS_DAT, &orig->capabilities);
}

/**
 * batadv_dat_hash_free() - free the local DAT hash table
 * @bat_priv: the bat priv with all the soft interface information
 */
static void batadv_dat_hash_free(struct batadv_priv *bat_priv)
{
        if (!bat_priv->dat.hash)
                return;

        __batadv_dat_purge(bat_priv, NULL);

        batadv_hash_destroy(bat_priv->dat.hash);

        bat_priv->dat.hash = NULL;
}

/**
 * batadv_dat_init() - initialise the DAT internals
 * @bat_priv: the bat priv with all the soft interface information
 *
 * Return: 0 in case of success, a negative error code otherwise
 */
int batadv_dat_init(struct batadv_priv *bat_priv)
{
        if (bat_priv->dat.hash)
                return 0;

        bat_priv->dat.hash = batadv_hash_new(1024);

        if (!bat_priv->dat.hash)
                return -ENOMEM;

        INIT_DELAYED_WORK(&bat_priv->dat.work, batadv_dat_purge);
        batadv_dat_start_timer(bat_priv);

        batadv_tvlv_handler_register(bat_priv, batadv_dat_tvlv_ogm_handler_v1,
                                     NULL, NULL, BATADV_TVLV_DAT, 1,
                                     BATADV_TVLV_HANDLER_OGM_CIFNOTFND);
        batadv_dat_tvlv_container_update(bat_priv);
        return 0;
}

/**
 * batadv_dat_free() - free the DAT internals
 * @bat_priv: the bat priv with all the soft interface information
 */
void batadv_dat_free(struct batadv_priv *bat_priv)
{
        batadv_tvlv_container_unregister(bat_priv, BATADV_TVLV_DAT, 1);
        batadv_tvlv_handler_unregister(bat_priv, BATADV_TVLV_DAT, 1);

        cancel_delayed_work_sync(&bat_priv->dat.work);

        batadv_dat_hash_free(bat_priv);
}

/**
 * batadv_dat_cache_dump_entry() - dump one entry of the DAT cache table to a
 *  netlink socket
 * @msg: buffer for the message
 * @portid: netlink port
 * @cb: Control block containing additional options
 * @dat_entry: entry to dump
 *
 * Return: 0 or error code.
 */
static int
batadv_dat_cache_dump_entry(struct sk_buff *msg, u32 portid,
                            struct netlink_callback *cb,
                            struct batadv_dat_entry *dat_entry)
{
        int msecs;
        void *hdr;

        hdr = genlmsg_put(msg, portid, cb->nlh->nlmsg_seq,
                          &batadv_netlink_family, NLM_F_MULTI,
                          BATADV_CMD_GET_DAT_CACHE);
        if (!hdr)
                return -ENOBUFS;

        genl_dump_check_consistent(cb, hdr);

        msecs = jiffies_to_msecs(jiffies - dat_entry->last_update);

        if (nla_put_in_addr(msg, BATADV_ATTR_DAT_CACHE_IP4ADDRESS,
                            dat_entry->ip) ||
            nla_put(msg, BATADV_ATTR_DAT_CACHE_HWADDRESS, ETH_ALEN,
                    dat_entry->mac_addr) ||
            nla_put_u16(msg, BATADV_ATTR_DAT_CACHE_VID, dat_entry->vid) ||
            nla_put_u32(msg, BATADV_ATTR_LAST_SEEN_MSECS, msecs)) {
                genlmsg_cancel(msg, hdr);
                return -EMSGSIZE;
        }

        genlmsg_end(msg, hdr);
        return 0;
}

/**
 * batadv_dat_cache_dump_bucket() - dump one bucket of the DAT cache table to
 *  a netlink socket
 * @msg: buffer for the message
 * @portid: netlink port
 * @cb: Control block containing additional options
 * @hash: hash to dump
 * @bucket: bucket index to dump
 * @idx_skip: How many entries to skip
 *
 * Return: 0 or error code.
 */
static int
batadv_dat_cache_dump_bucket(struct sk_buff *msg, u32 portid,
                             struct netlink_callback *cb,
                             struct batadv_hashtable *hash, unsigned int bucket,
                             int *idx_skip)
{
        struct batadv_dat_entry *dat_entry;
        int idx = 0;

        spin_lock_bh(&hash->list_locks[bucket]);
        cb->seq = atomic_read(&hash->generation) << 1 | 1;

        hlist_for_each_entry(dat_entry, &hash->table[bucket], hash_entry) {
                if (idx < *idx_skip)
                        goto skip;

                if (batadv_dat_cache_dump_entry(msg, portid, cb, dat_entry)) {
                        spin_unlock_bh(&hash->list_locks[bucket]);
                        *idx_skip = idx;

                        return -EMSGSIZE;
                }

skip:
                idx++;
        }
        spin_unlock_bh(&hash->list_locks[bucket]);

        return 0;
}

/**
 * batadv_dat_cache_dump() - dump DAT cache table to a netlink socket
 * @msg: buffer for the message
 * @cb: callback structure containing arguments
 *
 * Return: message length.
 */
int batadv_dat_cache_dump(struct sk_buff *msg, struct netlink_callback *cb)
{
        struct batadv_hard_iface *primary_if = NULL;
        int portid = NETLINK_CB(cb->skb).portid;
        struct net *net = sock_net(cb->skb->sk);
        struct net_device *soft_iface;
        struct batadv_hashtable *hash;
        struct batadv_priv *bat_priv;
        int bucket = cb->args[0];
        int idx = cb->args[1];
        int ifindex;
        int ret = 0;

        ifindex = batadv_netlink_get_ifindex(cb->nlh,
                                             BATADV_ATTR_MESH_IFINDEX);
        if (!ifindex)
                return -EINVAL;

        soft_iface = dev_get_by_index(net, ifindex);
        if (!soft_iface || !batadv_softif_is_valid(soft_iface)) {
                ret = -ENODEV;
                goto out;
        }

        bat_priv = netdev_priv(soft_iface);
        hash = bat_priv->dat.hash;

        primary_if = batadv_primary_if_get_selected(bat_priv);
        if (!primary_if || primary_if->if_status != BATADV_IF_ACTIVE) {
                ret = -ENOENT;
                goto out;
        }

        while (bucket < hash->size) {
                if (batadv_dat_cache_dump_bucket(msg, portid, cb, hash, bucket,
                                                 &idx))
                        break;

                bucket++;
                idx = 0;
        }

        cb->args[0] = bucket;
        cb->args[1] = idx;

        ret = msg->len;

out:
        batadv_hardif_put(primary_if);

        dev_put(soft_iface);

        return ret;
}

/**
 * batadv_arp_get_type() - parse an ARP packet and gets the type
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: packet to analyse
 * @hdr_size: size of the possible header before the ARP packet in the skb
 *
 * Return: the ARP type if the skb contains a valid ARP packet, 0 otherwise.
 */
static u16 batadv_arp_get_type(struct batadv_priv *bat_priv,
                               struct sk_buff *skb, int hdr_size)
{
        struct arphdr *arphdr;
        struct ethhdr *ethhdr;
        __be32 ip_src, ip_dst;
        u8 *hw_src, *hw_dst;
        u16 type = 0;

        /* pull the ethernet header */
        if (unlikely(!pskb_may_pull(skb, hdr_size + ETH_HLEN)))
                goto out;

        ethhdr = (struct ethhdr *)(skb->data + hdr_size);

        if (ethhdr->h_proto != htons(ETH_P_ARP))
                goto out;

        /* pull the ARP payload */
        if (unlikely(!pskb_may_pull(skb, hdr_size + ETH_HLEN +
                                    arp_hdr_len(skb->dev))))
                goto out;

        arphdr = (struct arphdr *)(skb->data + hdr_size + ETH_HLEN);

        /* check whether the ARP packet carries a valid IP information */
        if (arphdr->ar_hrd != htons(ARPHRD_ETHER))
                goto out;

        if (arphdr->ar_pro != htons(ETH_P_IP))
                goto out;

        if (arphdr->ar_hln != ETH_ALEN)
                goto out;

        if (arphdr->ar_pln != 4)
                goto out;

        /* Check for bad reply/request. If the ARP message is not sane, DAT
         * will simply ignore it
         */
        ip_src = batadv_arp_ip_src(skb, hdr_size);
        ip_dst = batadv_arp_ip_dst(skb, hdr_size);
        if (ipv4_is_loopback(ip_src) || ipv4_is_multicast(ip_src) ||
            ipv4_is_loopback(ip_dst) || ipv4_is_multicast(ip_dst) ||
            ipv4_is_zeronet(ip_src) || ipv4_is_lbcast(ip_src) ||
            ipv4_is_zeronet(ip_dst) || ipv4_is_lbcast(ip_dst))
                goto out;

        hw_src = batadv_arp_hw_src(skb, hdr_size);
        if (is_zero_ether_addr(hw_src) || is_multicast_ether_addr(hw_src))
                goto out;

        /* don't care about the destination MAC address in ARP requests */
        if (arphdr->ar_op != htons(ARPOP_REQUEST)) {
                hw_dst = batadv_arp_hw_dst(skb, hdr_size);
                if (is_zero_ether_addr(hw_dst) ||
                    is_multicast_ether_addr(hw_dst))
                        goto out;
        }

        type = ntohs(arphdr->ar_op);
out:
        return type;
}

/**
 * batadv_dat_get_vid() - extract the VLAN identifier from skb if any
 * @skb: the buffer containing the packet to extract the VID from
 * @hdr_size: the size of the batman-adv header encapsulating the packet
 *
 * Return: If the packet embedded in the skb is vlan tagged this function
 * returns the VID with the BATADV_VLAN_HAS_TAG flag. Otherwise BATADV_NO_FLAGS
 * is returned.
 */
static unsigned short batadv_dat_get_vid(struct sk_buff *skb, int *hdr_size)
{
        unsigned short vid;

        vid = batadv_get_vid(skb, *hdr_size);

        /* ARP parsing functions jump forward of hdr_size + ETH_HLEN.
         * If the header contained in the packet is a VLAN one (which is longer)
         * hdr_size is updated so that the functions will still skip the
         * correct amount of bytes.
         */
        if (vid & BATADV_VLAN_HAS_TAG)
                *hdr_size += VLAN_HLEN;

        return vid;
}

/**
 * batadv_dat_arp_create_reply() - create an ARP Reply
 * @bat_priv: the bat priv with all the soft interface information
 * @ip_src: ARP sender IP
 * @ip_dst: ARP target IP
 * @hw_src: Ethernet source and ARP sender MAC
 * @hw_dst: Ethernet destination and ARP target MAC
 * @vid: VLAN identifier (optional, set to zero otherwise)
 *
 * Creates an ARP Reply from the given values, optionally encapsulated in a
 * VLAN header.
 *
 * Return: An skb containing an ARP Reply.
 */
static struct sk_buff *
batadv_dat_arp_create_reply(struct batadv_priv *bat_priv, __be32 ip_src,
                            __be32 ip_dst, u8 *hw_src, u8 *hw_dst,
                            unsigned short vid)
{
        struct sk_buff *skb;

        skb = arp_create(ARPOP_REPLY, ETH_P_ARP, ip_dst, bat_priv->soft_iface,
                         ip_src, hw_dst, hw_src, hw_dst);
        if (!skb)
                return NULL;

        skb_reset_mac_header(skb);

        if (vid & BATADV_VLAN_HAS_TAG)
                skb = vlan_insert_tag(skb, htons(ETH_P_8021Q),
                                      vid & VLAN_VID_MASK);

        return skb;
}

/**
 * batadv_dat_snoop_outgoing_arp_request() - snoop the ARP request and try to
 * answer using DAT
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: packet to check
 *
 * Return: true if the message has been sent to the dht candidates, false
 * otherwise. In case of a positive return value the message has to be enqueued
 * to permit the fallback.
 */
bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv,
                                           struct sk_buff *skb)
{
        u16 type = 0;
        __be32 ip_dst, ip_src;
        u8 *hw_src;
        bool ret = false;
        struct batadv_dat_entry *dat_entry = NULL;
        struct sk_buff *skb_new;
        struct net_device *soft_iface = bat_priv->soft_iface;
        int hdr_size = 0;
        unsigned short vid;

        if (!atomic_read(&bat_priv->distributed_arp_table))
                goto out;

        vid = batadv_dat_get_vid(skb, &hdr_size);

        type = batadv_arp_get_type(bat_priv, skb, hdr_size);
        /* If the node gets an ARP_REQUEST it has to send a DHT_GET unicast
         * message to the selected DHT candidates
         */
        if (type != ARPOP_REQUEST)
                goto out;

        batadv_dbg_arp(bat_priv, skb, hdr_size, "Parsing outgoing ARP REQUEST");

        ip_src = batadv_arp_ip_src(skb, hdr_size);
        hw_src = batadv_arp_hw_src(skb, hdr_size);
        ip_dst = batadv_arp_ip_dst(skb, hdr_size);

        batadv_dat_entry_add(bat_priv, ip_src, hw_src, vid);

        dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_dst, vid);
        if (dat_entry) {
                /* If the ARP request is destined for a local client the local
                 * client will answer itself. DAT would only generate a
                 * duplicate packet.
                 *
                 * Moreover, if the soft-interface is enslaved into a bridge, an
                 * additional DAT answer may trigger kernel warnings about
                 * a packet coming from the wrong port.
                 */
                if (batadv_is_my_client(bat_priv, dat_entry->mac_addr, vid)) {
                        ret = true;
                        goto out;
                }

                /* If BLA is enabled, only send ARP replies if we have claimed
                 * the destination for the ARP request or if no one else of
                 * the backbone gws belonging to our backbone has claimed the
                 * destination.
                 */
                if (!batadv_bla_check_claim(bat_priv,
                                            dat_entry->mac_addr, vid)) {
                        batadv_dbg(BATADV_DBG_DAT, bat_priv,
                                   "Device %pM claimed by another backbone gw. Don't send ARP reply!",
                                   dat_entry->mac_addr);
                        ret = true;
                        goto out;
                }

                skb_new = batadv_dat_arp_create_reply(bat_priv, ip_dst, ip_src,
                                                      dat_entry->mac_addr,
                                                      hw_src, vid);
                if (!skb_new)
                        goto out;

                skb_new->protocol = eth_type_trans(skb_new, soft_iface);

                batadv_inc_counter(bat_priv, BATADV_CNT_RX);
                batadv_add_counter(bat_priv, BATADV_CNT_RX_BYTES,
                                   skb->len + ETH_HLEN + hdr_size);

                netif_rx(skb_new);
                batadv_dbg(BATADV_DBG_DAT, bat_priv, "ARP request replied locally\n");
                ret = true;
        } else {
                /* Send the request to the DHT */
                ret = batadv_dat_forward_data(bat_priv, skb, ip_dst, vid,
                                              BATADV_P_DAT_DHT_GET);
        }
out:
        batadv_dat_entry_put(dat_entry);
        return ret;
}

/**
 * batadv_dat_snoop_incoming_arp_request() - snoop the ARP request and try to
 * answer using the local DAT storage
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: packet to check
 * @hdr_size: size of the encapsulation header
 *
 * Return: true if the request has been answered, false otherwise.
 */
bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv,
                                           struct sk_buff *skb, int hdr_size)
{
        u16 type;
        __be32 ip_src, ip_dst;
        u8 *hw_src;
        struct sk_buff *skb_new;
        struct batadv_dat_entry *dat_entry = NULL;
        bool ret = false;
        unsigned short vid;
        int err;

        if (!atomic_read(&bat_priv->distributed_arp_table))
                goto out;

        vid = batadv_dat_get_vid(skb, &hdr_size);

        type = batadv_arp_get_type(bat_priv, skb, hdr_size);
        if (type != ARPOP_REQUEST)
                goto out;

        hw_src = batadv_arp_hw_src(skb, hdr_size);
        ip_src = batadv_arp_ip_src(skb, hdr_size);
        ip_dst = batadv_arp_ip_dst(skb, hdr_size);

        batadv_dbg_arp(bat_priv, skb, hdr_size, "Parsing incoming ARP REQUEST");

        batadv_dat_entry_add(bat_priv, ip_src, hw_src, vid);

        dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_dst, vid);
        if (!dat_entry)
                goto out;

        skb_new = batadv_dat_arp_create_reply(bat_priv, ip_dst, ip_src,
                                              dat_entry->mac_addr, hw_src, vid);
        if (!skb_new)
                goto out;

        /* To preserve backwards compatibility, the node has choose the outgoing
         * format based on the incoming request packet type. The assumption is
         * that a node not using the 4addr packet format doesn't support it.
         */
        if (hdr_size == sizeof(struct batadv_unicast_4addr_packet))
                err = batadv_send_skb_via_tt_4addr(bat_priv, skb_new,
                                                   BATADV_P_DAT_CACHE_REPLY,
                                                   NULL, vid);
        else
                err = batadv_send_skb_via_tt(bat_priv, skb_new, NULL, vid);

        if (err != NET_XMIT_DROP) {
                batadv_inc_counter(bat_priv, BATADV_CNT_DAT_CACHED_REPLY_TX);
                ret = true;
        }
out:
        batadv_dat_entry_put(dat_entry);
        if (ret)
                kfree_skb(skb);
        return ret;
}

/**
 * batadv_dat_snoop_outgoing_arp_reply() - snoop the ARP reply and fill the DHT
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: packet to check
 */
void batadv_dat_snoop_outgoing_arp_reply(struct batadv_priv *bat_priv,
                                         struct sk_buff *skb)
{
        u16 type;
        __be32 ip_src, ip_dst;
        u8 *hw_src, *hw_dst;
        int hdr_size = 0;
        unsigned short vid;

        if (!atomic_read(&bat_priv->distributed_arp_table))
                return;

        vid = batadv_dat_get_vid(skb, &hdr_size);

        type = batadv_arp_get_type(bat_priv, skb, hdr_size);
        if (type != ARPOP_REPLY)
                return;

        batadv_dbg_arp(bat_priv, skb, hdr_size, "Parsing outgoing ARP REPLY");

        hw_src = batadv_arp_hw_src(skb, hdr_size);
        ip_src = batadv_arp_ip_src(skb, hdr_size);
        hw_dst = batadv_arp_hw_dst(skb, hdr_size);
        ip_dst = batadv_arp_ip_dst(skb, hdr_size);

        batadv_dat_entry_add(bat_priv, ip_src, hw_src, vid);
        batadv_dat_entry_add(bat_priv, ip_dst, hw_dst, vid);

        /* Send the ARP reply to the candidates for both the IP addresses that
         * the node obtained from the ARP reply
         */
        batadv_dat_forward_data(bat_priv, skb, ip_src, vid,
                                BATADV_P_DAT_DHT_PUT);
        batadv_dat_forward_data(bat_priv, skb, ip_dst, vid,
                                BATADV_P_DAT_DHT_PUT);
}

/**
 * batadv_dat_snoop_incoming_arp_reply() - snoop the ARP reply and fill the
 *  local DAT storage only
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: packet to check
 * @hdr_size: size of the encapsulation header
 *
 * Return: true if the packet was snooped and consumed by DAT. False if the
 * packet has to be delivered to the interface
 */
bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv,
                                         struct sk_buff *skb, int hdr_size)
{
        struct batadv_dat_entry *dat_entry = NULL;
        u16 type;
        __be32 ip_src, ip_dst;
        u8 *hw_src, *hw_dst;
        bool dropped = false;
        unsigned short vid;

        if (!atomic_read(&bat_priv->distributed_arp_table))
                goto out;

        vid = batadv_dat_get_vid(skb, &hdr_size);

        type = batadv_arp_get_type(bat_priv, skb, hdr_size);
        if (type != ARPOP_REPLY)
                goto out;

        batadv_dbg_arp(bat_priv, skb, hdr_size, "Parsing incoming ARP REPLY");

        hw_src = batadv_arp_hw_src(skb, hdr_size);
        ip_src = batadv_arp_ip_src(skb, hdr_size);
        hw_dst = batadv_arp_hw_dst(skb, hdr_size);
        ip_dst = batadv_arp_ip_dst(skb, hdr_size);

        /* If ip_dst is already in cache and has the right mac address,
         * drop this frame if this ARP reply is destined for us because it's
         * most probably an ARP reply generated by another node of the DHT.
         * We have most probably received already a reply earlier. Delivering
         * this frame would lead to doubled receive of an ARP reply.
         */
        dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_src, vid);
        if (dat_entry && batadv_compare_eth(hw_src, dat_entry->mac_addr)) {
                batadv_dbg(BATADV_DBG_DAT, bat_priv, "Doubled ARP reply removed: ARP MSG = [src: %pM-%pI4 dst: %pM-%pI4]; dat_entry: %pM-%pI4\n",
                           hw_src, &ip_src, hw_dst, &ip_dst,
                           dat_entry->mac_addr,        &dat_entry->ip);
                dropped = true;
        }

        /* Update our internal cache with both the IP addresses the node got
         * within the ARP reply
         */
        batadv_dat_entry_add(bat_priv, ip_src, hw_src, vid);
        batadv_dat_entry_add(bat_priv, ip_dst, hw_dst, vid);

        if (dropped)
                goto out;

        /* If BLA is enabled, only forward ARP replies if we have claimed the
         * source of the ARP reply or if no one else of the same backbone has
         * already claimed that client. This prevents that different gateways
         * to the same backbone all forward the ARP reply leading to multiple
         * replies in the backbone.
         */
        if (!batadv_bla_check_claim(bat_priv, hw_src, vid)) {
                batadv_dbg(BATADV_DBG_DAT, bat_priv,
                           "Device %pM claimed by another backbone gw. Drop ARP reply.\n",
                           hw_src);
                dropped = true;
                goto out;
        }

        /* if this REPLY is directed to a client of mine, let's deliver the
         * packet to the interface
         */
        dropped = !batadv_is_my_client(bat_priv, hw_dst, vid);

        /* if this REPLY is sent on behalf of a client of mine, let's drop the
         * packet because the client will reply by itself
         */
        dropped |= batadv_is_my_client(bat_priv, hw_src, vid);
out:
        if (dropped)
                kfree_skb(skb);
        batadv_dat_entry_put(dat_entry);
        /* if dropped == false -> deliver to the interface */
        return dropped;
}

/**
 * batadv_dat_check_dhcp_ipudp() - check skb for IP+UDP headers valid for DHCP
 * @skb: the packet to check
 * @ip_src: a buffer to store the IPv4 source address in
 *
 * Checks whether the given skb has an IP and UDP header valid for a DHCP
 * message from a DHCP server. And if so, stores the IPv4 source address in
 * the provided buffer.
 *
 * Return: True if valid, false otherwise.
 */
static bool
batadv_dat_check_dhcp_ipudp(struct sk_buff *skb, __be32 *ip_src)
{
        unsigned int offset = skb_network_offset(skb);
        struct udphdr *udphdr, _udphdr;
        struct iphdr *iphdr, _iphdr;

        iphdr = skb_header_pointer(skb, offset, sizeof(_iphdr), &_iphdr);
        if (!iphdr || iphdr->version != 4 || iphdr->ihl * 4 < sizeof(_iphdr))
                return false;

        if (iphdr->protocol != IPPROTO_UDP)
                return false;

        offset += iphdr->ihl * 4;
        skb_set_transport_header(skb, offset);

        udphdr = skb_header_pointer(skb, offset, sizeof(_udphdr), &_udphdr);
        if (!udphdr || udphdr->source != htons(67))
                return false;

        *ip_src = get_unaligned(&iphdr->saddr);

        return true;
}

/**
 * batadv_dat_check_dhcp() - examine packet for valid DHCP message
 * @skb: the packet to check
 * @proto: ethernet protocol hint (behind a potential vlan)
 * @ip_src: a buffer to store the IPv4 source address in
 *
 * Checks whether the given skb is a valid DHCP packet. And if so, stores the
 * IPv4 source address in the provided buffer.
 *
 * Caller needs to ensure that the skb network header is set correctly.
 *
 * Return: If skb is a valid DHCP packet, then returns its op code
 * (e.g. BOOTREPLY vs. BOOTREQUEST). Otherwise returns -EINVAL.
 */
static int
batadv_dat_check_dhcp(struct sk_buff *skb, __be16 proto, __be32 *ip_src)
{
        __be32 *magic, _magic;
        unsigned int offset;
        struct {
                __u8 op;
                __u8 htype;
                __u8 hlen;
                __u8 hops;
        } *dhcp_h, _dhcp_h;

        if (proto != htons(ETH_P_IP))
                return -EINVAL;

        if (!batadv_dat_check_dhcp_ipudp(skb, ip_src))
                return -EINVAL;

        offset = skb_transport_offset(skb) + sizeof(struct udphdr);
        if (skb->len < offset + sizeof(struct batadv_dhcp_packet))
                return -EINVAL;

        dhcp_h = skb_header_pointer(skb, offset, sizeof(_dhcp_h), &_dhcp_h);
        if (!dhcp_h || dhcp_h->htype != BATADV_HTYPE_ETHERNET ||
            dhcp_h->hlen != ETH_ALEN)
                return -EINVAL;

        offset += offsetof(struct batadv_dhcp_packet, magic);

        magic = skb_header_pointer(skb, offset, sizeof(_magic), &_magic);
        if (!magic || get_unaligned(magic) != htonl(BATADV_DHCP_MAGIC))
                return -EINVAL;

        return dhcp_h->op;
}

/**
 * batadv_dat_get_dhcp_message_type() - get message type of a DHCP packet
 * @skb: the DHCP packet to parse
 *
 * Iterates over the DHCP options of the given DHCP packet to find a
 * DHCP Message Type option and parse it.
 *
 * Caller needs to ensure that the given skb is a valid DHCP packet and
 * that the skb transport header is set correctly.
 *
 * Return: The found DHCP message type value, if found. -EINVAL otherwise.
 */
static int batadv_dat_get_dhcp_message_type(struct sk_buff *skb)
{
        unsigned int offset = skb_transport_offset(skb) + sizeof(struct udphdr);
        u8 *type, _type;
        struct {
                u8 type;
                u8 len;
        } *tl, _tl;

        offset += sizeof(struct batadv_dhcp_packet);

        while ((tl = skb_header_pointer(skb, offset, sizeof(_tl), &_tl))) {
                if (tl->type == BATADV_DHCP_OPT_MSG_TYPE)
                        break;

                if (tl->type == BATADV_DHCP_OPT_END)
                        break;

                if (tl->type == BATADV_DHCP_OPT_PAD)
                        offset++;
                else
                        offset += tl->len + sizeof(_tl);
        }

        /* Option Overload Code not supported */
        if (!tl || tl->type != BATADV_DHCP_OPT_MSG_TYPE ||
            tl->len != sizeof(_type))
                return -EINVAL;

        offset += sizeof(_tl);

        type = skb_header_pointer(skb, offset, sizeof(_type), &_type);
        if (!type)
                return -EINVAL;

        return *type;
}

/**
 * batadv_dat_dhcp_get_yiaddr() - get yiaddr from a DHCP packet
 * @skb: the DHCP packet to parse
 * @buf: a buffer to store the yiaddr in
 *
 * Caller needs to ensure that the given skb is a valid DHCP packet and
 * that the skb transport header is set correctly.
 *
 * Return: True on success, false otherwise.
 */
static bool batadv_dat_dhcp_get_yiaddr(struct sk_buff *skb, __be32 *buf)
{
        unsigned int offset = skb_transport_offset(skb) + sizeof(struct udphdr);
        __be32 *yiaddr;

        offset += offsetof(struct batadv_dhcp_packet, yiaddr);
        yiaddr = skb_header_pointer(skb, offset, BATADV_DHCP_YIADDR_LEN, buf);

        if (!yiaddr)
                return false;

        if (yiaddr != buf)
                *buf = get_unaligned(yiaddr);

        return true;
}

/**
 * batadv_dat_get_dhcp_chaddr() - get chaddr from a DHCP packet
 * @skb: the DHCP packet to parse
 * @buf: a buffer to store the chaddr in
 *
 * Caller needs to ensure that the given skb is a valid DHCP packet and
 * that the skb transport header is set correctly.
 *
 * Return: True on success, false otherwise
 */
static bool batadv_dat_get_dhcp_chaddr(struct sk_buff *skb, u8 *buf)
{
        unsigned int offset = skb_transport_offset(skb) + sizeof(struct udphdr);
        u8 *chaddr;

        offset += offsetof(struct batadv_dhcp_packet, chaddr);
        chaddr = skb_header_pointer(skb, offset, BATADV_DHCP_CHADDR_LEN, buf);

        if (!chaddr)
                return false;

        if (chaddr != buf)
                memcpy(buf, chaddr, BATADV_DHCP_CHADDR_LEN);

        return true;
}

/**
 * batadv_dat_put_dhcp() - puts addresses from a DHCP packet into the DHT and
 *  DAT cache
 * @bat_priv: the bat priv with all the soft interface information
 * @chaddr: the DHCP client MAC address
 * @yiaddr: the DHCP client IP address
 * @hw_dst: the DHCP server MAC address
 * @ip_dst: the DHCP server IP address
 * @vid: VLAN identifier
 *
 * Adds given MAC/IP pairs to the local DAT cache and propagates them further
 * into the DHT.
 *
 * For the DHT propagation, client MAC + IP will appear as the ARP Reply
 * transmitter (and hw_dst/ip_dst as the target).
 */
static void batadv_dat_put_dhcp(struct batadv_priv *bat_priv, u8 *chaddr,
                                __be32 yiaddr, u8 *hw_dst, __be32 ip_dst,
                                unsigned short vid)
{
        struct sk_buff *skb;

        skb = batadv_dat_arp_create_reply(bat_priv, yiaddr, ip_dst, chaddr,
                                          hw_dst, vid);
        if (!skb)
                return;

        skb_set_network_header(skb, ETH_HLEN);

        batadv_dat_entry_add(bat_priv, yiaddr, chaddr, vid);
        batadv_dat_entry_add(bat_priv, ip_dst, hw_dst, vid);

        batadv_dat_forward_data(bat_priv, skb, yiaddr, vid,
                                BATADV_P_DAT_DHT_PUT);
        batadv_dat_forward_data(bat_priv, skb, ip_dst, vid,
                                BATADV_P_DAT_DHT_PUT);

        consume_skb(skb);

        batadv_dbg(BATADV_DBG_DAT, bat_priv,
                   "Snooped from outgoing DHCPACK (server address): %pI4, %pM (vid: %i)\n",
                   &ip_dst, hw_dst, batadv_print_vid(vid));
        batadv_dbg(BATADV_DBG_DAT, bat_priv,
                   "Snooped from outgoing DHCPACK (client address): %pI4, %pM (vid: %i)\n",
                   &yiaddr, chaddr, batadv_print_vid(vid));
}

/**
 * batadv_dat_check_dhcp_ack() - examine packet for valid DHCP message
 * @skb: the packet to check
 * @proto: ethernet protocol hint (behind a potential vlan)
 * @ip_src: a buffer to store the IPv4 source address in
 * @chaddr: a buffer to store the DHCP Client Hardware Address in
 * @yiaddr: a buffer to store the DHCP Your IP Address in
 *
 * Checks whether the given skb is a valid DHCPACK. And if so, stores the
 * IPv4 server source address (ip_src), client MAC address (chaddr) and client
 * IPv4 address (yiaddr) in the provided buffers.
 *
 * Caller needs to ensure that the skb network header is set correctly.
 *
 * Return: True if the skb is a valid DHCPACK. False otherwise.
 */
static bool
batadv_dat_check_dhcp_ack(struct sk_buff *skb, __be16 proto, __be32 *ip_src,
                          u8 *chaddr, __be32 *yiaddr)
{
        int type;

        type = batadv_dat_check_dhcp(skb, proto, ip_src);
        if (type != BATADV_BOOTREPLY)
                return false;

        type = batadv_dat_get_dhcp_message_type(skb);
        if (type != BATADV_DHCPACK)
                return false;

        if (!batadv_dat_dhcp_get_yiaddr(skb, yiaddr))
                return false;

        if (!batadv_dat_get_dhcp_chaddr(skb, chaddr))
                return false;

        return true;
}

/**
 * batadv_dat_snoop_outgoing_dhcp_ack() - snoop DHCPACK and fill DAT with it
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: the packet to snoop
 * @proto: ethernet protocol hint (behind a potential vlan)
 * @vid: VLAN identifier
 *
 * This function first checks whether the given skb is a valid DHCPACK. If
 * so then its source MAC and IP as well as its DHCP Client Hardware Address
 * field and DHCP Your IP Address field are added to the local DAT cache and
 * propagated into the DHT.
 *
 * Caller needs to ensure that the skb mac and network headers are set
 * correctly.
 */
void batadv_dat_snoop_outgoing_dhcp_ack(struct batadv_priv *bat_priv,
                                        struct sk_buff *skb,
                                        __be16 proto,
                                        unsigned short vid)
{
        u8 chaddr[BATADV_DHCP_CHADDR_LEN];
        __be32 ip_src, yiaddr;

        if (!atomic_read(&bat_priv->distributed_arp_table))
                return;

        if (!batadv_dat_check_dhcp_ack(skb, proto, &ip_src, chaddr, &yiaddr))
                return;

        batadv_dat_put_dhcp(bat_priv, chaddr, yiaddr, eth_hdr(skb)->h_source,
                            ip_src, vid);
}

/**
 * batadv_dat_snoop_incoming_dhcp_ack() - snoop DHCPACK and fill DAT cache
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: the packet to snoop
 * @hdr_size: header size, up to the tail of the batman-adv header
 *
 * This function first checks whether the given skb is a valid DHCPACK. If
 * so then its source MAC and IP as well as its DHCP Client Hardware Address
 * field and DHCP Your IP Address field are added to the local DAT cache.
 */
void batadv_dat_snoop_incoming_dhcp_ack(struct batadv_priv *bat_priv,
                                        struct sk_buff *skb, int hdr_size)
{
        u8 chaddr[BATADV_DHCP_CHADDR_LEN];
        struct ethhdr *ethhdr;
        __be32 ip_src, yiaddr;
        unsigned short vid;
        __be16 proto;
        u8 *hw_src;

        if (!atomic_read(&bat_priv->distributed_arp_table))
                return;

        if (unlikely(!pskb_may_pull(skb, hdr_size + ETH_HLEN)))
                return;

        ethhdr = (struct ethhdr *)(skb->data + hdr_size);
        skb_set_network_header(skb, hdr_size + ETH_HLEN);
        proto = ethhdr->h_proto;

        if (!batadv_dat_check_dhcp_ack(skb, proto, &ip_src, chaddr, &yiaddr))
                return;

        hw_src = ethhdr->h_source;
        vid = batadv_dat_get_vid(skb, &hdr_size);

        batadv_dat_entry_add(bat_priv, yiaddr, chaddr, vid);
        batadv_dat_entry_add(bat_priv, ip_src, hw_src, vid);

        batadv_dbg(BATADV_DBG_DAT, bat_priv,
                   "Snooped from incoming DHCPACK (server address): %pI4, %pM (vid: %i)\n",
                   &ip_src, hw_src, batadv_print_vid(vid));
        batadv_dbg(BATADV_DBG_DAT, bat_priv,
                   "Snooped from incoming DHCPACK (client address): %pI4, %pM (vid: %i)\n",
                   &yiaddr, chaddr, batadv_print_vid(vid));
}

/**
 * batadv_dat_drop_broadcast_packet() - check if an ARP request has to be
 *  dropped (because the node has already obtained the reply via DAT) or not
 * @bat_priv: the bat priv with all the soft interface information
 * @forw_packet: the broadcast packet
 *
 * Return: true if the node can drop the packet, false otherwise.
 */
bool batadv_dat_drop_broadcast_packet(struct batadv_priv *bat_priv,
                                      struct batadv_forw_packet *forw_packet)
{
        u16 type;
        __be32 ip_dst;
        struct batadv_dat_entry *dat_entry = NULL;
        bool ret = false;
        int hdr_size = sizeof(struct batadv_bcast_packet);
        unsigned short vid;

        if (!atomic_read(&bat_priv->distributed_arp_table))
                goto out;

        /* If this packet is an ARP_REQUEST and the node already has the
         * information that it is going to ask, then the packet can be dropped
         */
        if (batadv_forw_packet_is_rebroadcast(forw_packet))
                goto out;

        vid = batadv_dat_get_vid(forw_packet->skb, &hdr_size);

        type = batadv_arp_get_type(bat_priv, forw_packet->skb, hdr_size);
        if (type != ARPOP_REQUEST)
                goto out;

        ip_dst = batadv_arp_ip_dst(forw_packet->skb, hdr_size);
        dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_dst, vid);
        /* check if the node already got this entry */
        if (!dat_entry) {
                batadv_dbg(BATADV_DBG_DAT, bat_priv,
                           "ARP Request for %pI4: fallback\n", &ip_dst);
                goto out;
        }

        batadv_dbg(BATADV_DBG_DAT, bat_priv,
                   "ARP Request for %pI4: fallback prevented\n", &ip_dst);
        ret = true;

out:
        batadv_dat_entry_put(dat_entry);
        return ret;
}






















































































































































































































































































































   12 





























































































































































































































































































































































































































































































































































  702 












   12 
    4 










   12 















































































































































  241 



























































































































































 3155 



































































































  132 






























    3 

    3 

    3 















  264 















































    1 

   12 
























































































    2 






















    2 
    2 

































































































































































































































































































































































































    4 






































































































































































































































   17 


   31 








   12 


























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _NET_XFRM_H
#define _NET_XFRM_H

#include <linux/compiler.h>
#include <linux/xfrm.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/skbuff.h>
#include <linux/socket.h>
#include <linux/pfkeyv2.h>
#include <linux/ipsec.h>
#include <linux/in6.h>
#include <linux/mutex.h>
#include <linux/audit.h>
#include <linux/slab.h>
#include <linux/refcount.h>
#include <linux/sockptr.h>

#include <net/sock.h>
#include <net/dst.h>
#include <net/inet_dscp.h>
#include <net/ip.h>
#include <net/route.h>
#include <net/ipv6.h>
#include <net/ip6_fib.h>
#include <net/flow.h>
#include <net/gro_cells.h>

#include <linux/interrupt.h>

#ifdef CONFIG_XFRM_STATISTICS
#include <net/snmp.h>
#endif

#define XFRM_PROTO_ESP                50
#define XFRM_PROTO_AH                51
#define XFRM_PROTO_COMP                108
#define XFRM_PROTO_IPIP                4
#define XFRM_PROTO_IPV6                41
#define XFRM_PROTO_ROUTING        IPPROTO_ROUTING
#define XFRM_PROTO_DSTOPTS        IPPROTO_DSTOPTS

#define XFRM_ALIGN4(len)        (((len) + 3) & ~3)
#define XFRM_ALIGN8(len)        (((len) + 7) & ~7)
#define MODULE_ALIAS_XFRM_MODE(family, encap) \
        MODULE_ALIAS("xfrm-mode-" __stringify(family) "-" __stringify(encap))
#define MODULE_ALIAS_XFRM_TYPE(family, proto) \
        MODULE_ALIAS("xfrm-type-" __stringify(family) "-" __stringify(proto))
#define MODULE_ALIAS_XFRM_OFFLOAD_TYPE(family, proto) \
        MODULE_ALIAS("xfrm-offload-" __stringify(family) "-" __stringify(proto))

#ifdef CONFIG_XFRM_STATISTICS
#define XFRM_INC_STATS(net, field)        SNMP_INC_STATS((net)->mib.xfrm_statistics, field)
#define XFRM_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.xfrm_statistics, field, val)
#else
#define XFRM_INC_STATS(net, field)        ((void)(net))
#define XFRM_ADD_STATS(net, field, val) ((void)(net))
#endif


/* Organization of SPD aka "XFRM rules"
   ------------------------------------

   Basic objects:
   - policy rule, struct xfrm_policy (=SPD entry)
   - bundle of transformations, struct dst_entry == struct xfrm_dst (=SA bundle)
   - instance of a transformer, struct xfrm_state (=SA)
   - template to clone xfrm_state, struct xfrm_tmpl

   SPD is organized as hash table (for policies that meet minimum address prefix
   length setting, net->xfrm.policy_hthresh).  Other policies are stored in
   lists, sorted into rbtree ordered by destination and source address networks.
   See net/xfrm/xfrm_policy.c for details.

   (To be compatible with existing pfkeyv2 implementations,
   many rules with priority of 0x7fffffff are allowed to exist and
   such rules are ordered in an unpredictable way, thanks to bsd folks.)

   If "action" is "block", then we prohibit the flow, otherwise:
   if "xfrms_nr" is zero, the flow passes untransformed. Otherwise,
   policy entry has list of up to XFRM_MAX_DEPTH transformations,
   described by templates xfrm_tmpl. Each template is resolved
   to a complete xfrm_state (see below) and we pack bundle of transformations
   to a dst_entry returned to requester.

   dst -. xfrm  .-> xfrm_state #1
    |---. child .-> dst -. xfrm .-> xfrm_state #2
                     |---. child .-> dst -. xfrm .-> xfrm_state #3
                                      |---. child .-> NULL


   Resolution of xrfm_tmpl
   -----------------------
   Template contains:
   1. ->mode                Mode: transport or tunnel
   2. ->id.proto        Protocol: AH/ESP/IPCOMP
   3. ->id.daddr        Remote tunnel endpoint, ignored for transport mode.
      Q: allow to resolve security gateway?
   4. ->id.spi          If not zero, static SPI.
   5. ->saddr                Local tunnel endpoint, ignored for transport mode.
   6. ->algos                List of allowed algos. Plain bitmask now.
      Q: ealgos, aalgos, calgos. What a mess...
   7. ->share                Sharing mode.
      Q: how to implement private sharing mode? To add struct sock* to
      flow id?

   Having this template we search through SAD searching for entries
   with appropriate mode/proto/algo, permitted by selector.
   If no appropriate entry found, it is requested from key manager.

   PROBLEMS:
   Q: How to find all the bundles referring to a physical path for
      PMTU discovery? Seems, dst should contain list of all parents...
      and enter to infinite locking hierarchy disaster.
      No! It is easier, we will not search for them, let them find us.
      We add genid to each dst plus pointer to genid of raw IP route,
      pmtu disc will update pmtu on raw IP route and increase its genid.
      dst_check() will see this for top level and trigger resyncing
      metrics. Plus, it will be made via sk->sk_dst_cache. Solved.
 */

struct xfrm_state_walk {
        struct list_head        all;
        u8                        state;
        u8                        dying;
        u8                        proto;
        u32                        seq;
        struct xfrm_address_filter *filter;
};

enum {
        XFRM_DEV_OFFLOAD_IN = 1,
        XFRM_DEV_OFFLOAD_OUT,
        XFRM_DEV_OFFLOAD_FWD,
};

enum {
        XFRM_DEV_OFFLOAD_UNSPECIFIED,
        XFRM_DEV_OFFLOAD_CRYPTO,
        XFRM_DEV_OFFLOAD_PACKET,
};

enum {
        XFRM_DEV_OFFLOAD_FLAG_ACQ = 1,
};

struct xfrm_dev_offload {
        struct net_device        *dev;
        netdevice_tracker        dev_tracker;
        struct net_device        *real_dev;
        unsigned long                offload_handle;
        u8                        dir : 2;
        u8                        type : 2;
        u8                        flags : 2;
};

struct xfrm_mode {
        u8 encap;
        u8 family;
        u8 flags;
};

/* Flags for xfrm_mode. */
enum {
        XFRM_MODE_FLAG_TUNNEL = 1,
};

enum xfrm_replay_mode {
        XFRM_REPLAY_MODE_LEGACY,
        XFRM_REPLAY_MODE_BMP,
        XFRM_REPLAY_MODE_ESN,
};

/* Full description of state of transformer. */
struct xfrm_state {
        possible_net_t                xs_net;
        union {
                struct hlist_node        gclist;
                struct hlist_node        bydst;
        };
        union {
                struct hlist_node        dev_gclist;
                struct hlist_node        bysrc;
        };
        struct hlist_node        byspi;
        struct hlist_node        byseq;
        struct hlist_node        state_cache;
        struct hlist_node        state_cache_input;

        refcount_t                refcnt;
        spinlock_t                lock;

        u32                        pcpu_num;
        struct xfrm_id                id;
        struct xfrm_selector        sel;
        struct xfrm_mark        mark;
        u32                        if_id;
        u32                        tfcpad;

        u32                        genid;

        /* Key manager bits */
        struct xfrm_state_walk        km;

        /* Parameters of this state. */
        struct {
                u32                reqid;
                u8                mode;
                u8                replay_window;
                u8                aalgo, ealgo, calgo;
                u8                flags;
                u16                family;
                xfrm_address_t        saddr;
                int                header_len;
                int                trailer_len;
                u32                extra_flags;
                struct xfrm_mark        smark;
        } props;

        struct xfrm_lifetime_cfg lft;

        /* Data for transformer */
        struct xfrm_algo_auth        *aalg;
        struct xfrm_algo        *ealg;
        struct xfrm_algo        *calg;
        struct xfrm_algo_aead        *aead;
        const char                *geniv;

        /* mapping change rate limiting */
        __be16 new_mapping_sport;
        u32 new_mapping;        /* seconds */
        u32 mapping_maxage;        /* seconds for input SA */

        /* Data for encapsulator */
        struct xfrm_encap_tmpl        *encap;
        struct sock __rcu        *encap_sk;

        /* NAT keepalive */
        u32                        nat_keepalive_interval; /* seconds */
        time64_t                nat_keepalive_expiration;

        /* Data for care-of address */
        xfrm_address_t        *coaddr;

        /* IPComp needs an IPIP tunnel for handling uncompressed packets */
        struct xfrm_state        *tunnel;

        /* If a tunnel, number of users + 1 */
        atomic_t                tunnel_users;

        /* State for replay detection */
        struct xfrm_replay_state replay;
        struct xfrm_replay_state_esn *replay_esn;

        /* Replay detection state at the time we sent the last notification */
        struct xfrm_replay_state preplay;
        struct xfrm_replay_state_esn *preplay_esn;

        /* replay detection mode */
        enum xfrm_replay_mode    repl_mode;
        /* internal flag that only holds state for delayed aevent at the
         * moment
        */
        u32                        xflags;

        /* Replay detection notification settings */
        u32                        replay_maxage;
        u32                        replay_maxdiff;

        /* Replay detection notification timer */
        struct timer_list        rtimer;

        /* Statistics */
        struct xfrm_stats        stats;

        struct xfrm_lifetime_cur curlft;
        struct hrtimer                mtimer;

        struct xfrm_dev_offload xso;

        /* used to fix curlft->add_time when changing date */
        long                saved_tmo;

        /* Last used time */
        time64_t                lastused;

        struct page_frag xfrag;

        /* Reference to data common to all the instances of this
         * transformer. */
        const struct xfrm_type        *type;
        struct xfrm_mode        inner_mode;
        struct xfrm_mode        inner_mode_iaf;
        struct xfrm_mode        outer_mode;

        const struct xfrm_type_offload        *type_offload;

        /* Security context */
        struct xfrm_sec_ctx        *security;

        /* Private data of this transformer, format is opaque,
         * interpreted by xfrm_type methods. */
        void                        *data;
        u8                        dir;
};

static inline struct net *xs_net(struct xfrm_state *x)
{
        return read_pnet(&x->xs_net);
}

/* xflags - make enum if more show up */
#define XFRM_TIME_DEFER        1
#define XFRM_SOFT_EXPIRE 2

enum {
        XFRM_STATE_VOID,
        XFRM_STATE_ACQ,
        XFRM_STATE_VALID,
        XFRM_STATE_ERROR,
        XFRM_STATE_EXPIRED,
        XFRM_STATE_DEAD
};

/* callback structure passed from either netlink or pfkey */
struct km_event {
        union {
                u32 hard;
                u32 proto;
                u32 byid;
                u32 aevent;
                u32 type;
        } data;

        u32        seq;
        u32        portid;
        u32        event;
        struct net *net;
};

struct xfrm_if_decode_session_result {
        struct net *net;
        u32 if_id;
};

struct xfrm_if_cb {
        bool (*decode_session)(struct sk_buff *skb,
                               unsigned short family,
                               struct xfrm_if_decode_session_result *res);
};

void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb);
void xfrm_if_unregister_cb(void);

struct xfrm_dst_lookup_params {
        struct net *net;
        dscp_t dscp;
        int oif;
        xfrm_address_t *saddr;
        xfrm_address_t *daddr;
        u32 mark;
        __u8 ipproto;
        union flowi_uli uli;
};

struct net_device;
struct xfrm_type;
struct xfrm_dst;
struct xfrm_policy_afinfo {
        struct dst_ops                *dst_ops;
        struct dst_entry        *(*dst_lookup)(const struct xfrm_dst_lookup_params *params);
        int                        (*get_saddr)(xfrm_address_t *saddr,
                                             const struct xfrm_dst_lookup_params *params);
        int                        (*fill_dst)(struct xfrm_dst *xdst,
                                            struct net_device *dev,
                                            const struct flowi *fl);
        struct dst_entry        *(*blackhole_route)(struct net *net, struct dst_entry *orig);
};

int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family);
void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo);
void km_policy_notify(struct xfrm_policy *xp, int dir,
                      const struct km_event *c);
void km_state_notify(struct xfrm_state *x, const struct km_event *c);

struct xfrm_tmpl;
int km_query(struct xfrm_state *x, struct xfrm_tmpl *t,
             struct xfrm_policy *pol);
void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
int __xfrm_state_delete(struct xfrm_state *x);

struct xfrm_state_afinfo {
        u8                                family;
        u8                                proto;

        const struct xfrm_type_offload *type_offload_esp;

        const struct xfrm_type                *type_esp;
        const struct xfrm_type                *type_ipip;
        const struct xfrm_type                *type_ipip6;
        const struct xfrm_type                *type_comp;
        const struct xfrm_type                *type_ah;
        const struct xfrm_type                *type_routing;
        const struct xfrm_type                *type_dstopts;

        int                        (*output)(struct net *net, struct sock *sk, struct sk_buff *skb);
        int                        (*transport_finish)(struct sk_buff *skb,
                                                    int async);
        void                        (*local_error)(struct sk_buff *skb, u32 mtu);
};

int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo);
int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo);
struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family);
struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family);

struct xfrm_input_afinfo {
        u8                        family;
        bool                        is_ipip;
        int                        (*callback)(struct sk_buff *skb, u8 protocol,
                                            int err);
};

int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo);
int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo);

void xfrm_flush_gc(void);
void xfrm_state_delete_tunnel(struct xfrm_state *x);

struct xfrm_type {
        struct module                *owner;
        u8                        proto;
        u8                        flags;
#define XFRM_TYPE_NON_FRAGMENT        1
#define XFRM_TYPE_REPLAY_PROT        2
#define XFRM_TYPE_LOCAL_COADDR        4
#define XFRM_TYPE_REMOTE_COADDR        8

        int                        (*init_state)(struct xfrm_state *x,
                                              struct netlink_ext_ack *extack);
        void                        (*destructor)(struct xfrm_state *);
        int                        (*input)(struct xfrm_state *, struct sk_buff *skb);
        int                        (*output)(struct xfrm_state *, struct sk_buff *pskb);
        int                        (*reject)(struct xfrm_state *, struct sk_buff *,
                                          const struct flowi *);
};

int xfrm_register_type(const struct xfrm_type *type, unsigned short family);
void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family);

struct xfrm_type_offload {
        struct module        *owner;
        u8                proto;
        void                (*encap)(struct xfrm_state *, struct sk_buff *pskb);
        int                (*input_tail)(struct xfrm_state *x, struct sk_buff *skb);
        int                (*xmit)(struct xfrm_state *, struct sk_buff *pskb, netdev_features_t features);
};

int xfrm_register_type_offload(const struct xfrm_type_offload *type, unsigned short family);
void xfrm_unregister_type_offload(const struct xfrm_type_offload *type, unsigned short family);

static inline int xfrm_af2proto(unsigned int family)
{
        switch(family) {
        case AF_INET:
                return IPPROTO_IPIP;
        case AF_INET6:
                return IPPROTO_IPV6;
        default:
                return 0;
        }
}

static inline const struct xfrm_mode *xfrm_ip2inner_mode(struct xfrm_state *x, int ipproto)
{
        if ((ipproto == IPPROTO_IPIP && x->props.family == AF_INET) ||
            (ipproto == IPPROTO_IPV6 && x->props.family == AF_INET6))
                return &x->inner_mode;
        else
                return &x->inner_mode_iaf;
}

struct xfrm_tmpl {
/* id in template is interpreted as:
 * daddr - destination of tunnel, may be zero for transport mode.
 * spi   - zero to acquire spi. Not zero if spi is static, then
 *           daddr must be fixed too.
 * proto - AH/ESP/IPCOMP
 */
        struct xfrm_id                id;

/* Source address of tunnel. Ignored, if it is not a tunnel. */
        xfrm_address_t                saddr;

        unsigned short                encap_family;

        u32                        reqid;

/* Mode: transport, tunnel etc. */
        u8                        mode;

/* Sharing mode: unique, this session only, this user only etc. */
        u8                        share;

/* May skip this transfomration if no SA is found */
        u8                        optional;

/* Skip aalgos/ealgos/calgos checks. */
        u8                        allalgs;

/* Bit mask of algos allowed for acquisition */
        u32                        aalgos;
        u32                        ealgos;
        u32                        calgos;
};

#define XFRM_MAX_DEPTH                6
#define XFRM_MAX_OFFLOAD_DEPTH        1

struct xfrm_policy_walk_entry {
        struct list_head        all;
        u8                        dead;
};

struct xfrm_policy_walk {
        struct xfrm_policy_walk_entry walk;
        u8 type;
        u32 seq;
};

struct xfrm_policy_queue {
        struct sk_buff_head        hold_queue;
        struct timer_list        hold_timer;
        unsigned long                timeout;
};

/**
 *        struct xfrm_policy - xfrm policy
 *        @xp_net: network namespace the policy lives in
 *        @bydst: hlist node for SPD hash table or rbtree list
 *        @byidx: hlist node for index hash table
 *        @state_cache_list: hlist head for policy cached xfrm states
 *        @lock: serialize changes to policy structure members
 *        @refcnt: reference count, freed once it reaches 0
 *        @pos: kernel internal tie-breaker to determine age of policy
 *        @timer: timer
 *        @genid: generation, used to invalidate old policies
 *        @priority: priority, set by userspace
 *        @index:  policy index (autogenerated)
 *        @if_id: virtual xfrm interface id
 *        @mark: packet mark
 *        @selector: selector
 *        @lft: liftime configuration data
 *        @curlft: liftime state
 *        @walk: list head on pernet policy list
 *        @polq: queue to hold packets while aqcuire operaion in progress
 *        @bydst_reinsert: policy tree node needs to be merged
 *        @type: XFRM_POLICY_TYPE_MAIN or _SUB
 *        @action: XFRM_POLICY_ALLOW or _BLOCK
 *        @flags: XFRM_POLICY_LOCALOK, XFRM_POLICY_ICMP
 *        @xfrm_nr: number of used templates in @xfrm_vec
 *        @family: protocol family
 *        @security: SELinux security label
 *        @xfrm_vec: array of templates to resolve state
 *        @rcu: rcu head, used to defer memory release
 *        @xdo: hardware offload state
 */
struct xfrm_policy {
        possible_net_t                xp_net;
        struct hlist_node        bydst;
        struct hlist_node        byidx;

        struct hlist_head        state_cache_list;

        /* This lock only affects elements except for entry. */
        rwlock_t                lock;
        refcount_t                refcnt;
        u32                        pos;
        struct timer_list        timer;

        atomic_t                genid;
        u32                        priority;
        u32                        index;
        u32                        if_id;
        struct xfrm_mark        mark;
        struct xfrm_selector        selector;
        struct xfrm_lifetime_cfg lft;
        struct xfrm_lifetime_cur curlft;
        struct xfrm_policy_walk_entry walk;
        struct xfrm_policy_queue polq;
        bool                    bydst_reinsert;
        u8                        type;
        u8                        action;
        u8                        flags;
        u8                        xfrm_nr;
        u16                        family;
        struct xfrm_sec_ctx        *security;
        struct xfrm_tmpl               xfrm_vec[XFRM_MAX_DEPTH];
        struct rcu_head                rcu;

        struct xfrm_dev_offload xdo;
};

static inline struct net *xp_net(const struct xfrm_policy *xp)
{
        return read_pnet(&xp->xp_net);
}

struct xfrm_kmaddress {
        xfrm_address_t          local;
        xfrm_address_t          remote;
        u32                        reserved;
        u16                        family;
};

struct xfrm_migrate {
        xfrm_address_t                old_daddr;
        xfrm_address_t                old_saddr;
        xfrm_address_t                new_daddr;
        xfrm_address_t                new_saddr;
        u8                        proto;
        u8                        mode;
        u16                        reserved;
        u32                        reqid;
        u16                        old_family;
        u16                        new_family;
};

#define XFRM_KM_TIMEOUT                30
/* what happened */
#define XFRM_REPLAY_UPDATE        XFRM_AE_CR
#define XFRM_REPLAY_TIMEOUT        XFRM_AE_CE

/* default aevent timeout in units of 100ms */
#define XFRM_AE_ETIME                        10
/* Async Event timer multiplier */
#define XFRM_AE_ETH_M                        10
/* default seq threshold size */
#define XFRM_AE_SEQT_SIZE                2

struct xfrm_mgr {
        struct list_head        list;
        int                        (*notify)(struct xfrm_state *x, const struct km_event *c);
        int                        (*acquire)(struct xfrm_state *x, struct xfrm_tmpl *, struct xfrm_policy *xp);
        struct xfrm_policy        *(*compile_policy)(struct sock *sk, int opt, u8 *data, int len, int *dir);
        int                        (*new_mapping)(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport);
        int                        (*notify_policy)(struct xfrm_policy *x, int dir, const struct km_event *c);
        int                        (*report)(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr);
        int                        (*migrate)(const struct xfrm_selector *sel,
                                           u8 dir, u8 type,
                                           const struct xfrm_migrate *m,
                                           int num_bundles,
                                           const struct xfrm_kmaddress *k,
                                           const struct xfrm_encap_tmpl *encap);
        bool                        (*is_alive)(const struct km_event *c);
};

void xfrm_register_km(struct xfrm_mgr *km);
void xfrm_unregister_km(struct xfrm_mgr *km);

struct xfrm_tunnel_skb_cb {
        union {
                struct inet_skb_parm h4;
                struct inet6_skb_parm h6;
        } header;

        union {
                struct ip_tunnel *ip4;
                struct ip6_tnl *ip6;
        } tunnel;
};

#define XFRM_TUNNEL_SKB_CB(__skb) ((struct xfrm_tunnel_skb_cb *)&((__skb)->cb[0]))

/*
 * This structure is used for the duration where packets are being
 * transformed by IPsec.  As soon as the packet leaves IPsec the
 * area beyond the generic IP part may be overwritten.
 */
struct xfrm_skb_cb {
        struct xfrm_tunnel_skb_cb header;

        /* Sequence number for replay protection. */
        union {
                struct {
                        __u32 low;
                        __u32 hi;
                } output;
                struct {
                        __be32 low;
                        __be32 hi;
                } input;
        } seq;
};

#define XFRM_SKB_CB(__skb) ((struct xfrm_skb_cb *)&((__skb)->cb[0]))

/*
 * This structure is used by the afinfo prepare_input/prepare_output functions
 * to transmit header information to the mode input/output functions.
 */
struct xfrm_mode_skb_cb {
        struct xfrm_tunnel_skb_cb header;

        /* Copied from header for IPv4, always set to zero and DF for IPv6. */
        __be16 id;
        __be16 frag_off;

        /* IP header length (excluding options or extension headers). */
        u8 ihl;

        /* TOS for IPv4, class for IPv6. */
        u8 tos;

        /* TTL for IPv4, hop limitfor IPv6. */
        u8 ttl;

        /* Protocol for IPv4, NH for IPv6. */
        u8 protocol;

        /* Option length for IPv4, zero for IPv6. */
        u8 optlen;

        /* Used by IPv6 only, zero for IPv4. */
        u8 flow_lbl[3];
};

#define XFRM_MODE_SKB_CB(__skb) ((struct xfrm_mode_skb_cb *)&((__skb)->cb[0]))

/*
 * This structure is used by the input processing to locate the SPI and
 * related information.
 */
struct xfrm_spi_skb_cb {
        struct xfrm_tunnel_skb_cb header;

        unsigned int daddroff;
        unsigned int family;
        __be32 seq;
};

#define XFRM_SPI_SKB_CB(__skb) ((struct xfrm_spi_skb_cb *)&((__skb)->cb[0]))

#ifdef CONFIG_AUDITSYSCALL
static inline struct audit_buffer *xfrm_audit_start(const char *op)
{
        struct audit_buffer *audit_buf = NULL;

        if (audit_enabled == AUDIT_OFF)
                return NULL;
        audit_buf = audit_log_start(audit_context(), GFP_ATOMIC,
                                    AUDIT_MAC_IPSEC_EVENT);
        if (audit_buf == NULL)
                return NULL;
        audit_log_format(audit_buf, "op=%s", op);
        return audit_buf;
}

static inline void xfrm_audit_helper_usrinfo(bool task_valid,
                                             struct audit_buffer *audit_buf)
{
        const unsigned int auid = from_kuid(&init_user_ns, task_valid ?
                                            audit_get_loginuid(current) :
                                            INVALID_UID);
        const unsigned int ses = task_valid ? audit_get_sessionid(current) :
                AUDIT_SID_UNSET;

        audit_log_format(audit_buf, " auid=%u ses=%u", auid, ses);
        audit_log_task_context(audit_buf);
}

void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid);
void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
                              bool task_valid);
void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid);
void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid);
void xfrm_audit_state_replay_overflow(struct xfrm_state *x,
                                      struct sk_buff *skb);
void xfrm_audit_state_replay(struct xfrm_state *x, struct sk_buff *skb,
                             __be32 net_seq);
void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family);
void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family, __be32 net_spi,
                               __be32 net_seq);
void xfrm_audit_state_icvfail(struct xfrm_state *x, struct sk_buff *skb,
                              u8 proto);
#else

static inline void xfrm_audit_policy_add(struct xfrm_policy *xp, int result,
                                         bool task_valid)
{
}

static inline void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
                                            bool task_valid)
{
}

static inline void xfrm_audit_state_add(struct xfrm_state *x, int result,
                                        bool task_valid)
{
}

static inline void xfrm_audit_state_delete(struct xfrm_state *x, int result,
                                           bool task_valid)
{
}

static inline void xfrm_audit_state_replay_overflow(struct xfrm_state *x,
                                             struct sk_buff *skb)
{
}

static inline void xfrm_audit_state_replay(struct xfrm_state *x,
                                           struct sk_buff *skb, __be32 net_seq)
{
}

static inline void xfrm_audit_state_notfound_simple(struct sk_buff *skb,
                                      u16 family)
{
}

static inline void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family,
                                      __be32 net_spi, __be32 net_seq)
{
}

static inline void xfrm_audit_state_icvfail(struct xfrm_state *x,
                                     struct sk_buff *skb, u8 proto)
{
}
#endif /* CONFIG_AUDITSYSCALL */

static inline void xfrm_pol_hold(struct xfrm_policy *policy)
{
        if (likely(policy != NULL))
                refcount_inc(&policy->refcnt);
}

void xfrm_policy_destroy(struct xfrm_policy *policy);

static inline void xfrm_pol_put(struct xfrm_policy *policy)
{
        if (refcount_dec_and_test(&policy->refcnt))
                xfrm_policy_destroy(policy);
}

static inline void xfrm_pols_put(struct xfrm_policy **pols, int npols)
{
        int i;
        for (i = npols - 1; i >= 0; --i)
                xfrm_pol_put(pols[i]);
}

void __xfrm_state_destroy(struct xfrm_state *, bool);

static inline void __xfrm_state_put(struct xfrm_state *x)
{
        refcount_dec(&x->refcnt);
}

static inline void xfrm_state_put(struct xfrm_state *x)
{
        if (refcount_dec_and_test(&x->refcnt))
                __xfrm_state_destroy(x, false);
}

static inline void xfrm_state_put_sync(struct xfrm_state *x)
{
        if (refcount_dec_and_test(&x->refcnt))
                __xfrm_state_destroy(x, true);
}

static inline void xfrm_state_hold(struct xfrm_state *x)
{
        refcount_inc(&x->refcnt);
}

static inline bool addr_match(const void *token1, const void *token2,
                              unsigned int prefixlen)
{
        const __be32 *a1 = token1;
        const __be32 *a2 = token2;
        unsigned int pdw;
        unsigned int pbi;

        pdw = prefixlen >> 5;          /* num of whole u32 in prefix */
        pbi = prefixlen &  0x1f;  /* num of bits in incomplete u32 in prefix */

        if (pdw)
                if (memcmp(a1, a2, pdw << 2))
                        return false;

        if (pbi) {
                __be32 mask;

                mask = htonl((0xffffffff) << (32 - pbi));

                if ((a1[pdw] ^ a2[pdw]) & mask)
                        return false;
        }

        return true;
}

static inline bool addr4_match(__be32 a1, __be32 a2, u8 prefixlen)
{
        /* C99 6.5.7 (3): u32 << 32 is undefined behaviour */
        if (sizeof(long) == 4 && prefixlen == 0)
                return true;
        return !((a1 ^ a2) & htonl(~0UL << (32 - prefixlen)));
}

static __inline__
__be16 xfrm_flowi_sport(const struct flowi *fl, const union flowi_uli *uli)
{
        __be16 port;
        switch(fl->flowi_proto) {
        case IPPROTO_TCP:
        case IPPROTO_UDP:
        case IPPROTO_UDPLITE:
        case IPPROTO_SCTP:
                port = uli->ports.sport;
                break;
        case IPPROTO_ICMP:
        case IPPROTO_ICMPV6:
                port = htons(uli->icmpt.type);
                break;
        case IPPROTO_MH:
                port = htons(uli->mht.type);
                break;
        case IPPROTO_GRE:
                port = htons(ntohl(uli->gre_key) >> 16);
                break;
        default:
                port = 0;        /*XXX*/
        }
        return port;
}

static __inline__
__be16 xfrm_flowi_dport(const struct flowi *fl, const union flowi_uli *uli)
{
        __be16 port;
        switch(fl->flowi_proto) {
        case IPPROTO_TCP:
        case IPPROTO_UDP:
        case IPPROTO_UDPLITE:
        case IPPROTO_SCTP:
                port = uli->ports.dport;
                break;
        case IPPROTO_ICMP:
        case IPPROTO_ICMPV6:
                port = htons(uli->icmpt.code);
                break;
        case IPPROTO_GRE:
                port = htons(ntohl(uli->gre_key) & 0xffff);
                break;
        default:
                port = 0;        /*XXX*/
        }
        return port;
}

bool xfrm_selector_match(const struct xfrm_selector *sel,
                         const struct flowi *fl, unsigned short family);

#ifdef CONFIG_SECURITY_NETWORK_XFRM
/*        If neither has a context --> match
 *         Otherwise, both must have a context and the sids, doi, alg must match
 */
static inline bool xfrm_sec_ctx_match(struct xfrm_sec_ctx *s1, struct xfrm_sec_ctx *s2)
{
        return ((!s1 && !s2) ||
                (s1 && s2 &&
                 (s1->ctx_sid == s2->ctx_sid) &&
                 (s1->ctx_doi == s2->ctx_doi) &&
                 (s1->ctx_alg == s2->ctx_alg)));
}
#else
static inline bool xfrm_sec_ctx_match(struct xfrm_sec_ctx *s1, struct xfrm_sec_ctx *s2)
{
        return true;
}
#endif

/* A struct encoding bundle of transformations to apply to some set of flow.
 *
 * xdst->child points to the next element of bundle.
 * dst->xfrm  points to an instanse of transformer.
 *
 * Due to unfortunate limitations of current routing cache, which we
 * have no time to fix, it mirrors struct rtable and bound to the same
 * routing key, including saddr,daddr. However, we can have many of
 * bundles differing by session id. All the bundles grow from a parent
 * policy rule.
 */
struct xfrm_dst {
        union {
                struct dst_entry        dst;
                struct rtable                rt;
                struct rt6_info                rt6;
        } u;
        struct dst_entry *route;
        struct dst_entry *child;
        struct dst_entry *path;
        struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
        int num_pols, num_xfrms;
        u32 xfrm_genid;
        u32 policy_genid;
        u32 route_mtu_cached;
        u32 child_mtu_cached;
        u32 route_cookie;
        u32 path_cookie;
};

static inline struct dst_entry *xfrm_dst_path(const struct dst_entry *dst)
{
#ifdef CONFIG_XFRM
        if (dst->xfrm || (dst->flags & DST_XFRM_QUEUE)) {
                const struct xfrm_dst *xdst = (const struct xfrm_dst *) dst;

                return xdst->path;
        }
#endif
        return (struct dst_entry *) dst;
}

static inline struct dst_entry *xfrm_dst_child(const struct dst_entry *dst)
{
#ifdef CONFIG_XFRM
        if (dst->xfrm || (dst->flags & DST_XFRM_QUEUE)) {
                struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
                return xdst->child;
        }
#endif
        return NULL;
}

#ifdef CONFIG_XFRM
static inline void xfrm_dst_set_child(struct xfrm_dst *xdst, struct dst_entry *child)
{
        xdst->child = child;
}

static inline void xfrm_dst_destroy(struct xfrm_dst *xdst)
{
        xfrm_pols_put(xdst->pols, xdst->num_pols);
        dst_release(xdst->route);
        if (likely(xdst->u.dst.xfrm))
                xfrm_state_put(xdst->u.dst.xfrm);
}
#endif

void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev);

struct xfrm_if_parms {
        int link;                /* ifindex of underlying L2 interface */
        u32 if_id;                /* interface identifier */
        bool collect_md;
};

struct xfrm_if {
        struct xfrm_if __rcu *next;        /* next interface in list */
        struct net_device *dev;                /* virtual device associated with interface */
        struct net *net;                /* netns for packet i/o */
        struct xfrm_if_parms p;                /* interface parms */

        struct gro_cells gro_cells;
};

struct xfrm_offload {
        /* Output sequence number for replay protection on offloading. */
        struct {
                __u32 low;
                __u32 hi;
        } seq;

        __u32                        flags;
#define        SA_DELETE_REQ                1
#define        CRYPTO_DONE                2
#define        CRYPTO_NEXT_DONE        4
#define        CRYPTO_FALLBACK                8
#define        XFRM_GSO_SEGMENT        16
#define        XFRM_GRO                32
/* 64 is free */
#define        XFRM_DEV_RESUME                128
#define        XFRM_XMIT                256

        __u32                        status;
#define CRYPTO_SUCCESS                                1
#define CRYPTO_GENERIC_ERROR                        2
#define CRYPTO_TRANSPORT_AH_AUTH_FAILED                4
#define CRYPTO_TRANSPORT_ESP_AUTH_FAILED        8
#define CRYPTO_TUNNEL_AH_AUTH_FAILED                16
#define CRYPTO_TUNNEL_ESP_AUTH_FAILED                32
#define CRYPTO_INVALID_PACKET_SYNTAX                64
#define CRYPTO_INVALID_PROTOCOL                        128

        /* Used to keep whole l2 header for transport mode GRO */
        __u32                        orig_mac_len;

        __u8                        proto;
        __u8                        inner_ipproto;
};

struct sec_path {
        int                        len;
        int                        olen;
        int                        verified_cnt;

        struct xfrm_state        *xvec[XFRM_MAX_DEPTH];
        struct xfrm_offload        ovec[XFRM_MAX_OFFLOAD_DEPTH];
};

struct sec_path *secpath_set(struct sk_buff *skb);

static inline void
secpath_reset(struct sk_buff *skb)
{
#ifdef CONFIG_XFRM
        skb_ext_del(skb, SKB_EXT_SEC_PATH);
#endif
}

static inline int
xfrm_addr_any(const xfrm_address_t *addr, unsigned short family)
{
        switch (family) {
        case AF_INET:
                return addr->a4 == 0;
        case AF_INET6:
                return ipv6_addr_any(&addr->in6);
        }
        return 0;
}

static inline int
__xfrm4_state_addr_cmp(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x)
{
        return        (tmpl->saddr.a4 &&
                 tmpl->saddr.a4 != x->props.saddr.a4);
}

static inline int
__xfrm6_state_addr_cmp(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x)
{
        return        (!ipv6_addr_any((struct in6_addr*)&tmpl->saddr) &&
                 !ipv6_addr_equal((struct in6_addr *)&tmpl->saddr, (struct in6_addr*)&x->props.saddr));
}

static inline int
xfrm_state_addr_cmp(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x, unsigned short family)
{
        switch (family) {
        case AF_INET:
                return __xfrm4_state_addr_cmp(tmpl, x);
        case AF_INET6:
                return __xfrm6_state_addr_cmp(tmpl, x);
        }
        return !0;
}

#ifdef CONFIG_XFRM
static inline struct xfrm_state *xfrm_input_state(struct sk_buff *skb)
{
        struct sec_path *sp = skb_sec_path(skb);

        return sp->xvec[sp->len - 1];
}
#endif

static inline struct xfrm_offload *xfrm_offload(struct sk_buff *skb)
{
#ifdef CONFIG_XFRM
        struct sec_path *sp = skb_sec_path(skb);

        if (!sp || !sp->olen || sp->len != sp->olen)
                return NULL;

        return &sp->ovec[sp->olen - 1];
#else
        return NULL;
#endif
}

#ifdef CONFIG_XFRM
int __xfrm_policy_check(struct sock *, int dir, struct sk_buff *skb,
                        unsigned short family);

static inline bool __xfrm_check_nopolicy(struct net *net, struct sk_buff *skb,
                                         int dir)
{
        if (!net->xfrm.policy_count[dir] && !secpath_exists(skb))
                return net->xfrm.policy_default[dir] == XFRM_USERPOLICY_ACCEPT;

        return false;
}

static inline bool __xfrm_check_dev_nopolicy(struct sk_buff *skb,
                                             int dir, unsigned short family)
{
        if (dir != XFRM_POLICY_OUT && family == AF_INET) {
                /* same dst may be used for traffic originating from
                 * devices with different policy settings.
                 */
                return IPCB(skb)->flags & IPSKB_NOPOLICY;
        }
        return skb_dst(skb) && (skb_dst(skb)->flags & DST_NOPOLICY);
}

static inline int __xfrm_policy_check2(struct sock *sk, int dir,
                                       struct sk_buff *skb,
                                       unsigned int family, int reverse)
{
        struct net *net = dev_net(skb->dev);
        int ndir = dir | (reverse ? XFRM_POLICY_MASK + 1 : 0);
        struct xfrm_offload *xo = xfrm_offload(skb);
        struct xfrm_state *x;

        if (sk && sk->sk_policy[XFRM_POLICY_IN])
                return __xfrm_policy_check(sk, ndir, skb, family);

        if (xo) {
                x = xfrm_input_state(skb);
                if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
                        return (xo->flags & CRYPTO_DONE) &&
                               (xo->status & CRYPTO_SUCCESS);
        }

        return __xfrm_check_nopolicy(net, skb, dir) ||
               __xfrm_check_dev_nopolicy(skb, dir, family) ||
               __xfrm_policy_check(sk, ndir, skb, family);
}

static inline int xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, unsigned short family)
{
        return __xfrm_policy_check2(sk, dir, skb, family, 0);
}

static inline int xfrm4_policy_check(struct sock *sk, int dir, struct sk_buff *skb)
{
        return xfrm_policy_check(sk, dir, skb, AF_INET);
}

static inline int xfrm6_policy_check(struct sock *sk, int dir, struct sk_buff *skb)
{
        return xfrm_policy_check(sk, dir, skb, AF_INET6);
}

static inline int xfrm4_policy_check_reverse(struct sock *sk, int dir,
                                             struct sk_buff *skb)
{
        return __xfrm_policy_check2(sk, dir, skb, AF_INET, 1);
}

static inline int xfrm6_policy_check_reverse(struct sock *sk, int dir,
                                             struct sk_buff *skb)
{
        return __xfrm_policy_check2(sk, dir, skb, AF_INET6, 1);
}

int __xfrm_decode_session(struct net *net, struct sk_buff *skb, struct flowi *fl,
                          unsigned int family, int reverse);

static inline int xfrm_decode_session(struct net *net, struct sk_buff *skb, struct flowi *fl,
                                      unsigned int family)
{
        return __xfrm_decode_session(net, skb, fl, family, 0);
}

static inline int xfrm_decode_session_reverse(struct net *net, struct sk_buff *skb,
                                              struct flowi *fl,
                                              unsigned int family)
{
        return __xfrm_decode_session(net, skb, fl, family, 1);
}

int __xfrm_route_forward(struct sk_buff *skb, unsigned short family);

static inline int xfrm_route_forward(struct sk_buff *skb, unsigned short family)
{
        struct net *net = dev_net(skb->dev);

        if (!net->xfrm.policy_count[XFRM_POLICY_OUT] &&
            net->xfrm.policy_default[XFRM_POLICY_OUT] == XFRM_USERPOLICY_ACCEPT)
                return true;

        return (skb_dst(skb)->flags & DST_NOXFRM) ||
               __xfrm_route_forward(skb, family);
}

static inline int xfrm4_route_forward(struct sk_buff *skb)
{
        return xfrm_route_forward(skb, AF_INET);
}

static inline int xfrm6_route_forward(struct sk_buff *skb)
{
        return xfrm_route_forward(skb, AF_INET6);
}

int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk);

static inline int xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk)
{
        if (!sk_fullsock(osk))
                return 0;
        sk->sk_policy[0] = NULL;
        sk->sk_policy[1] = NULL;
        if (unlikely(osk->sk_policy[0] || osk->sk_policy[1]))
                return __xfrm_sk_clone_policy(sk, osk);
        return 0;
}

int xfrm_policy_delete(struct xfrm_policy *pol, int dir);

static inline void xfrm_sk_free_policy(struct sock *sk)
{
        struct xfrm_policy *pol;

        pol = rcu_dereference_protected(sk->sk_policy[0], 1);
        if (unlikely(pol != NULL)) {
                xfrm_policy_delete(pol, XFRM_POLICY_MAX);
                sk->sk_policy[0] = NULL;
        }
        pol = rcu_dereference_protected(sk->sk_policy[1], 1);
        if (unlikely(pol != NULL)) {
                xfrm_policy_delete(pol, XFRM_POLICY_MAX+1);
                sk->sk_policy[1] = NULL;
        }
}

#else

static inline void xfrm_sk_free_policy(struct sock *sk) {}
static inline int xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk) { return 0; }
static inline int xfrm6_route_forward(struct sk_buff *skb) { return 1; }
static inline int xfrm4_route_forward(struct sk_buff *skb) { return 1; }
static inline int xfrm6_policy_check(struct sock *sk, int dir, struct sk_buff *skb)
{
        return 1;
}
static inline int xfrm4_policy_check(struct sock *sk, int dir, struct sk_buff *skb)
{
        return 1;
}
static inline int xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, unsigned short family)
{
        return 1;
}
static inline int xfrm_decode_session_reverse(struct net *net, struct sk_buff *skb,
                                              struct flowi *fl,
                                              unsigned int family)
{
        return -ENOSYS;
}
static inline int xfrm4_policy_check_reverse(struct sock *sk, int dir,
                                             struct sk_buff *skb)
{
        return 1;
}
static inline int xfrm6_policy_check_reverse(struct sock *sk, int dir,
                                             struct sk_buff *skb)
{
        return 1;
}
#endif

static __inline__
xfrm_address_t *xfrm_flowi_daddr(const struct flowi *fl, unsigned short family)
{
        switch (family){
        case AF_INET:
                return (xfrm_address_t *)&fl->u.ip4.daddr;
        case AF_INET6:
                return (xfrm_address_t *)&fl->u.ip6.daddr;
        }
        return NULL;
}

static __inline__
xfrm_address_t *xfrm_flowi_saddr(const struct flowi *fl, unsigned short family)
{
        switch (family){
        case AF_INET:
                return (xfrm_address_t *)&fl->u.ip4.saddr;
        case AF_INET6:
                return (xfrm_address_t *)&fl->u.ip6.saddr;
        }
        return NULL;
}

static __inline__
void xfrm_flowi_addr_get(const struct flowi *fl,
                         xfrm_address_t *saddr, xfrm_address_t *daddr,
                         unsigned short family)
{
        switch(family) {
        case AF_INET:
                memcpy(&saddr->a4, &fl->u.ip4.saddr, sizeof(saddr->a4));
                memcpy(&daddr->a4, &fl->u.ip4.daddr, sizeof(daddr->a4));
                break;
        case AF_INET6:
                saddr->in6 = fl->u.ip6.saddr;
                daddr->in6 = fl->u.ip6.daddr;
                break;
        }
}

static __inline__ int
__xfrm4_state_addr_check(const struct xfrm_state *x,
                         const xfrm_address_t *daddr, const xfrm_address_t *saddr)
{
        if (daddr->a4 == x->id.daddr.a4 &&
            (saddr->a4 == x->props.saddr.a4 || !saddr->a4 || !x->props.saddr.a4))
                return 1;
        return 0;
}

static __inline__ int
__xfrm6_state_addr_check(const struct xfrm_state *x,
                         const xfrm_address_t *daddr, const xfrm_address_t *saddr)
{
        if (ipv6_addr_equal((struct in6_addr *)daddr, (struct in6_addr *)&x->id.daddr) &&
            (ipv6_addr_equal((struct in6_addr *)saddr, (struct in6_addr *)&x->props.saddr) ||
             ipv6_addr_any((struct in6_addr *)saddr) ||
             ipv6_addr_any((struct in6_addr *)&x->props.saddr)))
                return 1;
        return 0;
}

static __inline__ int
xfrm_state_addr_check(const struct xfrm_state *x,
                      const xfrm_address_t *daddr, const xfrm_address_t *saddr,
                      unsigned short family)
{
        switch (family) {
        case AF_INET:
                return __xfrm4_state_addr_check(x, daddr, saddr);
        case AF_INET6:
                return __xfrm6_state_addr_check(x, daddr, saddr);
        }
        return 0;
}

static __inline__ int
xfrm_state_addr_flow_check(const struct xfrm_state *x, const struct flowi *fl,
                           unsigned short family)
{
        switch (family) {
        case AF_INET:
                return __xfrm4_state_addr_check(x,
                                                (const xfrm_address_t *)&fl->u.ip4.daddr,
                                                (const xfrm_address_t *)&fl->u.ip4.saddr);
        case AF_INET6:
                return __xfrm6_state_addr_check(x,
                                                (const xfrm_address_t *)&fl->u.ip6.daddr,
                                                (const xfrm_address_t *)&fl->u.ip6.saddr);
        }
        return 0;
}

static inline int xfrm_state_kern(const struct xfrm_state *x)
{
        return atomic_read(&x->tunnel_users);
}

static inline bool xfrm_id_proto_valid(u8 proto)
{
        switch (proto) {
        case IPPROTO_AH:
        case IPPROTO_ESP:
        case IPPROTO_COMP:
#if IS_ENABLED(CONFIG_IPV6)
        case IPPROTO_ROUTING:
        case IPPROTO_DSTOPTS:
#endif
                return true;
        default:
                return false;
        }
}

/* IPSEC_PROTO_ANY only matches 3 IPsec protocols, 0 could match all. */
static inline int xfrm_id_proto_match(u8 proto, u8 userproto)
{
        return (!userproto || proto == userproto ||
                (userproto == IPSEC_PROTO_ANY && (proto == IPPROTO_AH ||
                                                  proto == IPPROTO_ESP ||
                                                  proto == IPPROTO_COMP)));
}

/*
 * xfrm algorithm information
 */
struct xfrm_algo_aead_info {
        char *geniv;
        u16 icv_truncbits;
};

struct xfrm_algo_auth_info {
        u16 icv_truncbits;
        u16 icv_fullbits;
};

struct xfrm_algo_encr_info {
        char *geniv;
        u16 blockbits;
        u16 defkeybits;
};

struct xfrm_algo_comp_info {
        u16 threshold;
};

struct xfrm_algo_desc {
        char *name;
        char *compat;
        u8 available:1;
        u8 pfkey_supported:1;
        union {
                struct xfrm_algo_aead_info aead;
                struct xfrm_algo_auth_info auth;
                struct xfrm_algo_encr_info encr;
                struct xfrm_algo_comp_info comp;
        } uinfo;
        struct sadb_alg desc;
};

/* XFRM protocol handlers.  */
struct xfrm4_protocol {
        int (*handler)(struct sk_buff *skb);
        int (*input_handler)(struct sk_buff *skb, int nexthdr, __be32 spi,
                             int encap_type);
        int (*cb_handler)(struct sk_buff *skb, int err);
        int (*err_handler)(struct sk_buff *skb, u32 info);

        struct xfrm4_protocol __rcu *next;
        int priority;
};

struct xfrm6_protocol {
        int (*handler)(struct sk_buff *skb);
        int (*input_handler)(struct sk_buff *skb, int nexthdr, __be32 spi,
                             int encap_type);
        int (*cb_handler)(struct sk_buff *skb, int err);
        int (*err_handler)(struct sk_buff *skb, struct inet6_skb_parm *opt,
                           u8 type, u8 code, int offset, __be32 info);

        struct xfrm6_protocol __rcu *next;
        int priority;
};

/* XFRM tunnel handlers.  */
struct xfrm_tunnel {
        int (*handler)(struct sk_buff *skb);
        int (*cb_handler)(struct sk_buff *skb, int err);
        int (*err_handler)(struct sk_buff *skb, u32 info);

        struct xfrm_tunnel __rcu *next;
        int priority;
};

struct xfrm6_tunnel {
        int (*handler)(struct sk_buff *skb);
        int (*cb_handler)(struct sk_buff *skb, int err);
        int (*err_handler)(struct sk_buff *skb, struct inet6_skb_parm *opt,
                           u8 type, u8 code, int offset, __be32 info);
        struct xfrm6_tunnel __rcu *next;
        int priority;
};

void xfrm_init(void);
void xfrm4_init(void);
int xfrm_state_init(struct net *net);
void xfrm_state_fini(struct net *net);
void xfrm4_state_init(void);
void xfrm4_protocol_init(void);
#ifdef CONFIG_XFRM
int xfrm6_init(void);
void xfrm6_fini(void);
int xfrm6_state_init(void);
void xfrm6_state_fini(void);
int xfrm6_protocol_init(void);
void xfrm6_protocol_fini(void);
#else
static inline int xfrm6_init(void)
{
        return 0;
}
static inline void xfrm6_fini(void)
{
        ;
}
#endif

#ifdef CONFIG_XFRM_STATISTICS
int xfrm_proc_init(struct net *net);
void xfrm_proc_fini(struct net *net);
#endif

int xfrm_sysctl_init(struct net *net);
#ifdef CONFIG_SYSCTL
void xfrm_sysctl_fini(struct net *net);
#else
static inline void xfrm_sysctl_fini(struct net *net)
{
}
#endif

void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto,
                          struct xfrm_address_filter *filter);
int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk,
                    int (*func)(struct xfrm_state *, int, void*), void *);
void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net);
struct xfrm_state *xfrm_state_alloc(struct net *net);
void xfrm_state_free(struct xfrm_state *x);
struct xfrm_state *xfrm_state_find(const xfrm_address_t *daddr,
                                   const xfrm_address_t *saddr,
                                   const struct flowi *fl,
                                   struct xfrm_tmpl *tmpl,
                                   struct xfrm_policy *pol, int *err,
                                   unsigned short family, u32 if_id);
struct xfrm_state *xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id,
                                       xfrm_address_t *daddr,
                                       xfrm_address_t *saddr,
                                       unsigned short family,
                                       u8 mode, u8 proto, u32 reqid);
struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi,
                                              unsigned short family);
int xfrm_state_check_expire(struct xfrm_state *x);
void xfrm_state_update_stats(struct net *net);
#ifdef CONFIG_XFRM_OFFLOAD
static inline void xfrm_dev_state_update_stats(struct xfrm_state *x)
{
        struct xfrm_dev_offload *xdo = &x->xso;
        struct net_device *dev = READ_ONCE(xdo->dev);

        if (dev && dev->xfrmdev_ops &&
            dev->xfrmdev_ops->xdo_dev_state_update_stats)
                dev->xfrmdev_ops->xdo_dev_state_update_stats(x);

}
#else
static inline void xfrm_dev_state_update_stats(struct xfrm_state *x) {}
#endif
void xfrm_state_insert(struct xfrm_state *x);
int xfrm_state_add(struct xfrm_state *x);
int xfrm_state_update(struct xfrm_state *x);
struct xfrm_state *xfrm_state_lookup(struct net *net, u32 mark,
                                     const xfrm_address_t *daddr, __be32 spi,
                                     u8 proto, unsigned short family);
struct xfrm_state *xfrm_input_state_lookup(struct net *net, u32 mark,
                                           const xfrm_address_t *daddr,
                                           __be32 spi, u8 proto,
                                           unsigned short family);
struct xfrm_state *xfrm_state_lookup_byaddr(struct net *net, u32 mark,
                                            const xfrm_address_t *daddr,
                                            const xfrm_address_t *saddr,
                                            u8 proto,
                                            unsigned short family);
#ifdef CONFIG_XFRM_SUB_POLICY
void xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n,
                    unsigned short family);
void xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n,
                     unsigned short family);
#else
static inline void xfrm_tmpl_sort(struct xfrm_tmpl **d, struct xfrm_tmpl **s,
                                  int n, unsigned short family)
{
}

static inline void xfrm_state_sort(struct xfrm_state **d, struct xfrm_state **s,
                                   int n, unsigned short family)
{
}
#endif

struct xfrmk_sadinfo {
        u32 sadhcnt; /* current hash bkts */
        u32 sadhmcnt; /* max allowed hash bkts */
        u32 sadcnt; /* current running count */
};

struct xfrmk_spdinfo {
        u32 incnt;
        u32 outcnt;
        u32 fwdcnt;
        u32 inscnt;
        u32 outscnt;
        u32 fwdscnt;
        u32 spdhcnt;
        u32 spdhmcnt;
};

struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq, u32 pcpu_num);
int xfrm_state_delete(struct xfrm_state *x);
int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync);
int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid);
int xfrm_dev_policy_flush(struct net *net, struct net_device *dev,
                          bool task_valid);
void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si);
void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si);
u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq);
int xfrm_init_replay(struct xfrm_state *x, struct netlink_ext_ack *extack);
u32 xfrm_state_mtu(struct xfrm_state *x, int mtu);
int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload,
                      struct netlink_ext_ack *extack);
int xfrm_init_state(struct xfrm_state *x);
int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type);
int xfrm_input_resume(struct sk_buff *skb, int nexthdr);
int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
                         int (*finish)(struct net *, struct sock *,
                                       struct sk_buff *));
int xfrm_trans_queue(struct sk_buff *skb,
                     int (*finish)(struct net *, struct sock *,
                                   struct sk_buff *));
int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err);
int xfrm_output(struct sock *sk, struct sk_buff *skb);

#if IS_ENABLED(CONFIG_NET_PKTGEN)
int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb);
#endif

void xfrm_local_error(struct sk_buff *skb, int mtu);
int xfrm4_rcv_encap(struct sk_buff *skb, int nexthdr, __be32 spi,
                    int encap_type);
int xfrm4_transport_finish(struct sk_buff *skb, int async);
int xfrm4_rcv(struct sk_buff *skb);

static inline int xfrm4_rcv_spi(struct sk_buff *skb, int nexthdr, __be32 spi)
{
        XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL;
        XFRM_SPI_SKB_CB(skb)->family = AF_INET;
        XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr);
        return xfrm_input(skb, nexthdr, spi, 0);
}

int xfrm4_output(struct net *net, struct sock *sk, struct sk_buff *skb);
int xfrm4_protocol_register(struct xfrm4_protocol *handler, unsigned char protocol);
int xfrm4_protocol_deregister(struct xfrm4_protocol *handler, unsigned char protocol);
int xfrm4_tunnel_register(struct xfrm_tunnel *handler, unsigned short family);
int xfrm4_tunnel_deregister(struct xfrm_tunnel *handler, unsigned short family);
void xfrm4_local_error(struct sk_buff *skb, u32 mtu);
int xfrm6_rcv_spi(struct sk_buff *skb, int nexthdr, __be32 spi,
                  struct ip6_tnl *t);
int xfrm6_rcv_encap(struct sk_buff *skb, int nexthdr, __be32 spi,
                    int encap_type);
int xfrm6_transport_finish(struct sk_buff *skb, int async);
int xfrm6_rcv_tnl(struct sk_buff *skb, struct ip6_tnl *t);
int xfrm6_rcv(struct sk_buff *skb);
int xfrm6_input_addr(struct sk_buff *skb, xfrm_address_t *daddr,
                     xfrm_address_t *saddr, u8 proto);
void xfrm6_local_error(struct sk_buff *skb, u32 mtu);
int xfrm6_protocol_register(struct xfrm6_protocol *handler, unsigned char protocol);
int xfrm6_protocol_deregister(struct xfrm6_protocol *handler, unsigned char protocol);
int xfrm6_tunnel_register(struct xfrm6_tunnel *handler, unsigned short family);
int xfrm6_tunnel_deregister(struct xfrm6_tunnel *handler, unsigned short family);
__be32 xfrm6_tunnel_alloc_spi(struct net *net, xfrm_address_t *saddr);
__be32 xfrm6_tunnel_spi_lookup(struct net *net, const xfrm_address_t *saddr);
int xfrm6_output(struct net *net, struct sock *sk, struct sk_buff *skb);

#ifdef CONFIG_XFRM
void xfrm6_local_rxpmtu(struct sk_buff *skb, u32 mtu);
int xfrm4_udp_encap_rcv(struct sock *sk, struct sk_buff *skb);
int xfrm6_udp_encap_rcv(struct sock *sk, struct sk_buff *skb);
struct sk_buff *xfrm4_gro_udp_encap_rcv(struct sock *sk, struct list_head *head,
                                        struct sk_buff *skb);
struct sk_buff *xfrm6_gro_udp_encap_rcv(struct sock *sk, struct list_head *head,
                                        struct sk_buff *skb);
int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval,
                     int optlen);
#else
static inline int xfrm_user_policy(struct sock *sk, int optname,
                                   sockptr_t optval, int optlen)
{
         return -ENOPROTOOPT;
}
#endif

struct dst_entry *__xfrm_dst_lookup(int family, const struct xfrm_dst_lookup_params *params);

struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp);

void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type);
int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
                     int (*func)(struct xfrm_policy *, int, int, void*),
                     void *);
void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net);
int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl);
struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net,
                                          const struct xfrm_mark *mark,
                                          u32 if_id, u8 type, int dir,
                                          struct xfrm_selector *sel,
                                          struct xfrm_sec_ctx *ctx, int delete,
                                          int *err);
struct xfrm_policy *xfrm_policy_byid(struct net *net,
                                     const struct xfrm_mark *mark, u32 if_id,
                                     u8 type, int dir, u32 id, int delete,
                                     int *err);
int xfrm_policy_flush(struct net *net, u8 type, bool task_valid);
void xfrm_policy_hash_rebuild(struct net *net);
u32 xfrm_get_acqseq(void);
int verify_spi_info(u8 proto, u32 min, u32 max, struct netlink_ext_ack *extack);
int xfrm_alloc_spi(struct xfrm_state *x, u32 minspi, u32 maxspi,
                   struct netlink_ext_ack *extack);
struct xfrm_state *xfrm_find_acq(struct net *net, const struct xfrm_mark *mark,
                                 u8 mode, u32 reqid, u32 if_id, u32 pcpu_num, u8 proto,
                                 const xfrm_address_t *daddr,
                                 const xfrm_address_t *saddr, int create,
                                 unsigned short family);
int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol);

#ifdef CONFIG_XFRM_MIGRATE
int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
               const struct xfrm_migrate *m, int num_bundles,
               const struct xfrm_kmaddress *k,
               const struct xfrm_encap_tmpl *encap);
struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
                                                u32 if_id);
struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
                                      struct xfrm_migrate *m,
                                      struct xfrm_encap_tmpl *encap);
int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
                 struct xfrm_migrate *m, int num_bundles,
                 struct xfrm_kmaddress *k, struct net *net,
                 struct xfrm_encap_tmpl *encap, u32 if_id,
                 struct netlink_ext_ack *extack);
#endif

int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport);
void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid);
int km_report(struct net *net, u8 proto, struct xfrm_selector *sel,
              xfrm_address_t *addr);

void xfrm_input_init(void);
int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq);

void xfrm_probe_algs(void);
int xfrm_count_pfkey_auth_supported(void);
int xfrm_count_pfkey_enc_supported(void);
struct xfrm_algo_desc *xfrm_aalg_get_byidx(unsigned int idx);
struct xfrm_algo_desc *xfrm_ealg_get_byidx(unsigned int idx);
struct xfrm_algo_desc *xfrm_aalg_get_byid(int alg_id);
struct xfrm_algo_desc *xfrm_ealg_get_byid(int alg_id);
struct xfrm_algo_desc *xfrm_calg_get_byid(int alg_id);
struct xfrm_algo_desc *xfrm_aalg_get_byname(const char *name, int probe);
struct xfrm_algo_desc *xfrm_ealg_get_byname(const char *name, int probe);
struct xfrm_algo_desc *xfrm_calg_get_byname(const char *name, int probe);
struct xfrm_algo_desc *xfrm_aead_get_byname(const char *name, int icv_len,
                                            int probe);

static inline bool xfrm6_addr_equal(const xfrm_address_t *a,
                                    const xfrm_address_t *b)
{
        return ipv6_addr_equal((const struct in6_addr *)a,
                               (const struct in6_addr *)b);
}

static inline bool xfrm_addr_equal(const xfrm_address_t *a,
                                   const xfrm_address_t *b,
                                   sa_family_t family)
{
        switch (family) {
        default:
        case AF_INET:
                return ((__force u32)a->a4 ^ (__force u32)b->a4) == 0;
        case AF_INET6:
                return xfrm6_addr_equal(a, b);
        }
}

static inline int xfrm_policy_id2dir(u32 index)
{
        return index & 7;
}

#ifdef CONFIG_XFRM
void xfrm_replay_advance(struct xfrm_state *x, __be32 net_seq);
int xfrm_replay_check(struct xfrm_state *x, struct sk_buff *skb, __be32 net_seq);
void xfrm_replay_notify(struct xfrm_state *x, int event);
int xfrm_replay_overflow(struct xfrm_state *x, struct sk_buff *skb);
int xfrm_replay_recheck(struct xfrm_state *x, struct sk_buff *skb, __be32 net_seq);

static inline int xfrm_aevent_is_on(struct net *net)
{
        struct sock *nlsk;
        int ret = 0;

        rcu_read_lock();
        nlsk = rcu_dereference(net->xfrm.nlsk);
        if (nlsk)
                ret = netlink_has_listeners(nlsk, XFRMNLGRP_AEVENTS);
        rcu_read_unlock();
        return ret;
}

static inline int xfrm_acquire_is_on(struct net *net)
{
        struct sock *nlsk;
        int ret = 0;

        rcu_read_lock();
        nlsk = rcu_dereference(net->xfrm.nlsk);
        if (nlsk)
                ret = netlink_has_listeners(nlsk, XFRMNLGRP_ACQUIRE);
        rcu_read_unlock();

        return ret;
}
#endif

static inline unsigned int aead_len(struct xfrm_algo_aead *alg)
{
        return sizeof(*alg) + ((alg->alg_key_len + 7) / 8);
}

static inline unsigned int xfrm_alg_len(const struct xfrm_algo *alg)
{
        return sizeof(*alg) + ((alg->alg_key_len + 7) / 8);
}

static inline unsigned int xfrm_alg_auth_len(const struct xfrm_algo_auth *alg)
{
        return sizeof(*alg) + ((alg->alg_key_len + 7) / 8);
}

static inline unsigned int xfrm_replay_state_esn_len(struct xfrm_replay_state_esn *replay_esn)
{
        return sizeof(*replay_esn) + replay_esn->bmp_len * sizeof(__u32);
}

#ifdef CONFIG_XFRM_MIGRATE
static inline int xfrm_replay_clone(struct xfrm_state *x,
                                     struct xfrm_state *orig)
{

        x->replay_esn = kmemdup(orig->replay_esn,
                                xfrm_replay_state_esn_len(orig->replay_esn),
                                GFP_KERNEL);
        if (!x->replay_esn)
                return -ENOMEM;
        x->preplay_esn = kmemdup(orig->preplay_esn,
                                 xfrm_replay_state_esn_len(orig->preplay_esn),
                                 GFP_KERNEL);
        if (!x->preplay_esn)
                return -ENOMEM;

        return 0;
}

static inline struct xfrm_algo_aead *xfrm_algo_aead_clone(struct xfrm_algo_aead *orig)
{
        return kmemdup(orig, aead_len(orig), GFP_KERNEL);
}


static inline struct xfrm_algo *xfrm_algo_clone(struct xfrm_algo *orig)
{
        return kmemdup(orig, xfrm_alg_len(orig), GFP_KERNEL);
}

static inline struct xfrm_algo_auth *xfrm_algo_auth_clone(struct xfrm_algo_auth *orig)
{
        return kmemdup(orig, xfrm_alg_auth_len(orig), GFP_KERNEL);
}

static inline void xfrm_states_put(struct xfrm_state **states, int n)
{
        int i;
        for (i = 0; i < n; i++)
                xfrm_state_put(*(states + i));
}

static inline void xfrm_states_delete(struct xfrm_state **states, int n)
{
        int i;
        for (i = 0; i < n; i++)
                xfrm_state_delete(*(states + i));
}
#endif

void __init xfrm_dev_init(void);

#ifdef CONFIG_XFRM_OFFLOAD
void xfrm_dev_resume(struct sk_buff *skb);
void xfrm_dev_backlog(struct softnet_data *sd);
struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again);
int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
                       struct xfrm_user_offload *xuo,
                       struct netlink_ext_ack *extack);
int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp,
                        struct xfrm_user_offload *xuo, u8 dir,
                        struct netlink_ext_ack *extack);
bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
void xfrm_dev_state_delete(struct xfrm_state *x);
void xfrm_dev_state_free(struct xfrm_state *x);

static inline void xfrm_dev_state_advance_esn(struct xfrm_state *x)
{
        struct xfrm_dev_offload *xso = &x->xso;
        struct net_device *dev = READ_ONCE(xso->dev);

        if (dev && dev->xfrmdev_ops->xdo_dev_state_advance_esn)
                dev->xfrmdev_ops->xdo_dev_state_advance_esn(x);
}

static inline bool xfrm_dst_offload_ok(struct dst_entry *dst)
{
        struct xfrm_state *x = dst->xfrm;
        struct xfrm_dst *xdst;

        if (!x || !x->type_offload)
                return false;

        xdst = (struct xfrm_dst *) dst;
        if (!x->xso.offload_handle && !xdst->child->xfrm)
                return true;
        if (x->xso.offload_handle && (x->xso.dev == xfrm_dst_path(dst)->dev) &&
            !xdst->child->xfrm)
                return true;

        return false;
}

static inline void xfrm_dev_policy_delete(struct xfrm_policy *x)
{
        struct xfrm_dev_offload *xdo = &x->xdo;
        struct net_device *dev = xdo->dev;

        if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_policy_delete)
                dev->xfrmdev_ops->xdo_dev_policy_delete(x);
}

static inline void xfrm_dev_policy_free(struct xfrm_policy *x)
{
        struct xfrm_dev_offload *xdo = &x->xdo;
        struct net_device *dev = xdo->dev;

        if (dev && dev->xfrmdev_ops) {
                if (dev->xfrmdev_ops->xdo_dev_policy_free)
                        dev->xfrmdev_ops->xdo_dev_policy_free(x);
                xdo->dev = NULL;
                netdev_put(dev, &xdo->dev_tracker);
        }
}
#else
static inline void xfrm_dev_resume(struct sk_buff *skb)
{
}

static inline void xfrm_dev_backlog(struct softnet_data *sd)
{
}

static inline struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)
{
        return skb;
}

static inline int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_user_offload *xuo, struct netlink_ext_ack *extack)
{
        return 0;
}

static inline void xfrm_dev_state_delete(struct xfrm_state *x)
{
}

static inline void xfrm_dev_state_free(struct xfrm_state *x)
{
}

static inline int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp,
                                      struct xfrm_user_offload *xuo, u8 dir,
                                      struct netlink_ext_ack *extack)
{
        return 0;
}

static inline void xfrm_dev_policy_delete(struct xfrm_policy *x)
{
}

static inline void xfrm_dev_policy_free(struct xfrm_policy *x)
{
}

static inline bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
{
        return false;
}

static inline void xfrm_dev_state_advance_esn(struct xfrm_state *x)
{
}

static inline bool xfrm_dst_offload_ok(struct dst_entry *dst)
{
        return false;
}
#endif

static inline int xfrm_mark_get(struct nlattr **attrs, struct xfrm_mark *m)
{
        if (attrs[XFRMA_MARK])
                memcpy(m, nla_data(attrs[XFRMA_MARK]), sizeof(struct xfrm_mark));
        else
                m->v = m->m = 0;

        return m->v & m->m;
}

static inline int xfrm_mark_put(struct sk_buff *skb, const struct xfrm_mark *m)
{
        int ret = 0;

        if (m->m | m->v)
                ret = nla_put(skb, XFRMA_MARK, sizeof(struct xfrm_mark), m);
        return ret;
}

static inline __u32 xfrm_smark_get(__u32 mark, struct xfrm_state *x)
{
        struct xfrm_mark *m = &x->props.smark;

        return (m->v & m->m) | (mark & ~m->m);
}

static inline int xfrm_if_id_put(struct sk_buff *skb, __u32 if_id)
{
        int ret = 0;

        if (if_id)
                ret = nla_put_u32(skb, XFRMA_IF_ID, if_id);
        return ret;
}

static inline int xfrm_tunnel_check(struct sk_buff *skb, struct xfrm_state *x,
                                    unsigned int family)
{
        bool tunnel = false;

        switch(family) {
        case AF_INET:
                if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
                        tunnel = true;
                break;
        case AF_INET6:
                if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
                        tunnel = true;
                break;
        }
        if (tunnel && !(x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL))
                return -EINVAL;

        return 0;
}

extern const int xfrm_msg_min[XFRM_NR_MSGTYPES];
extern const struct nla_policy xfrma_policy[XFRMA_MAX+1];

struct xfrm_translator {
        /* Allocate frag_list and put compat translation there */
        int (*alloc_compat)(struct sk_buff *skb, const struct nlmsghdr *src);

        /* Allocate nlmsg with 64-bit translaton of received 32-bit message */
        struct nlmsghdr *(*rcv_msg_compat)(const struct nlmsghdr *nlh,
                        int maxtype, const struct nla_policy *policy,
                        struct netlink_ext_ack *extack);

        /* Translate 32-bit user_policy from sockptr */
        int (*xlate_user_policy_sockptr)(u8 **pdata32, int optlen);

        struct module *owner;
};

#if IS_ENABLED(CONFIG_XFRM_USER_COMPAT)
extern int xfrm_register_translator(struct xfrm_translator *xtr);
extern int xfrm_unregister_translator(struct xfrm_translator *xtr);
extern struct xfrm_translator *xfrm_get_translator(void);
extern void xfrm_put_translator(struct xfrm_translator *xtr);
#else
static inline struct xfrm_translator *xfrm_get_translator(void)
{
        return NULL;
}
static inline void xfrm_put_translator(struct xfrm_translator *xtr)
{
}
#endif

#if IS_ENABLED(CONFIG_IPV6)
static inline bool xfrm6_local_dontfrag(const struct sock *sk)
{
        int proto;

        if (!sk || sk->sk_family != AF_INET6)
                return false;

        proto = sk->sk_protocol;
        if (proto == IPPROTO_UDP || proto == IPPROTO_RAW)
                return inet6_test_bit(DONTFRAG, sk);

        return false;
}
#endif

#if (IS_BUILTIN(CONFIG_XFRM_INTERFACE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \
    (IS_MODULE(CONFIG_XFRM_INTERFACE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES))

extern struct metadata_dst __percpu *xfrm_bpf_md_dst;

int register_xfrm_interface_bpf(void);

#else

static inline int register_xfrm_interface_bpf(void)
{
        return 0;
}

#endif

#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF)
int register_xfrm_state_bpf(void);
#else
static inline int register_xfrm_state_bpf(void)
{
        return 0;
}
#endif

int xfrm_nat_keepalive_init(unsigned short family);
void xfrm_nat_keepalive_fini(unsigned short family);
int xfrm_nat_keepalive_net_init(struct net *net);
int xfrm_nat_keepalive_net_fini(struct net *net);
void xfrm_nat_keepalive_state_updated(struct xfrm_state *x);

#endif        /* _NET_XFRM_H */




































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * sysctl.h: General linux system control interface
 *
 * Begun 24 March 1995, Stephen Tweedie
 *
 ****************************************************************
 ****************************************************************
 **
 **  WARNING:
 **  The values in this file are exported to user space via 
 **  the sysctl() binary interface.  Do *NOT* change the
 **  numbering of any existing values here, and do not change
 **  any numbers within any one set of values.  If you have to
 **  redefine an existing interface, use a new number for it.
 **  The kernel will then return -ENOTDIR to any application using
 **  the old binary interface.
 **
 ****************************************************************
 ****************************************************************
 */
#ifndef _LINUX_SYSCTL_H
#define _LINUX_SYSCTL_H

#include <linux/list.h>
#include <linux/rcupdate.h>
#include <linux/wait.h>
#include <linux/rbtree.h>
#include <linux/uidgid.h>
#include <uapi/linux/sysctl.h>

/* For the /proc/sys support */
struct completion;
struct ctl_table;
struct nsproxy;
struct ctl_table_root;
struct ctl_table_header;
struct ctl_dir;

/* Keep the same order as in fs/proc/proc_sysctl.c */
#define SYSCTL_ZERO                        ((void *)&sysctl_vals[0])
#define SYSCTL_ONE                        ((void *)&sysctl_vals[1])
#define SYSCTL_TWO                        ((void *)&sysctl_vals[2])
#define SYSCTL_THREE                        ((void *)&sysctl_vals[3])
#define SYSCTL_FOUR                        ((void *)&sysctl_vals[4])
#define SYSCTL_ONE_HUNDRED                ((void *)&sysctl_vals[5])
#define SYSCTL_TWO_HUNDRED                ((void *)&sysctl_vals[6])
#define SYSCTL_ONE_THOUSAND                ((void *)&sysctl_vals[7])
#define SYSCTL_THREE_THOUSAND                ((void *)&sysctl_vals[8])
#define SYSCTL_INT_MAX                        ((void *)&sysctl_vals[9])

/* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */
#define SYSCTL_MAXOLDUID                ((void *)&sysctl_vals[10])
#define SYSCTL_NEG_ONE                        ((void *)&sysctl_vals[11])

extern const int sysctl_vals[];

#define SYSCTL_LONG_ZERO        ((void *)&sysctl_long_vals[0])
#define SYSCTL_LONG_ONE                ((void *)&sysctl_long_vals[1])
#define SYSCTL_LONG_MAX                ((void *)&sysctl_long_vals[2])

extern const unsigned long sysctl_long_vals[];

typedef int proc_handler(const struct ctl_table *ctl, int write, void *buffer,
                size_t *lenp, loff_t *ppos);

int proc_dostring(const struct ctl_table *, int, void *, size_t *, loff_t *);
int proc_dobool(const struct ctl_table *table, int write, void *buffer,
                size_t *lenp, loff_t *ppos);
int proc_dointvec(const struct ctl_table *, int, void *, size_t *, loff_t *);
int proc_douintvec(const struct ctl_table *, int, void *, size_t *, loff_t *);
int proc_dointvec_minmax(const struct ctl_table *, int, void *, size_t *, loff_t *);
int proc_douintvec_minmax(const struct ctl_table *table, int write, void *buffer,
                size_t *lenp, loff_t *ppos);
int proc_dou8vec_minmax(const struct ctl_table *table, int write, void *buffer,
                        size_t *lenp, loff_t *ppos);
int proc_dointvec_jiffies(const struct ctl_table *, int, void *, size_t *, loff_t *);
int proc_dointvec_ms_jiffies_minmax(const struct ctl_table *table, int write,
                void *buffer, size_t *lenp, loff_t *ppos);
int proc_dointvec_userhz_jiffies(const struct ctl_table *, int, void *, size_t *,
                loff_t *);
int proc_dointvec_ms_jiffies(const struct ctl_table *, int, void *, size_t *,
                loff_t *);
int proc_doulongvec_minmax(const struct ctl_table *, int, void *, size_t *, loff_t *);
int proc_doulongvec_ms_jiffies_minmax(const struct ctl_table *table, int, void *,
                size_t *, loff_t *);
int proc_do_large_bitmap(const struct ctl_table *, int, void *, size_t *, loff_t *);
int proc_do_static_key(const struct ctl_table *table, int write, void *buffer,
                size_t *lenp, loff_t *ppos);

/*
 * Register a set of sysctl names by calling register_sysctl
 * with an initialised array of struct ctl_table's.
 *
 * sysctl names can be mirrored automatically under /proc/sys.  The
 * procname supplied controls /proc naming.
 *
 * The table's mode will be honoured for proc-fs access.
 *
 * Leaf nodes in the sysctl tree will be represented by a single file
 * under /proc; non-leaf nodes will be represented by directories.  A
 * null procname disables /proc mirroring at this node.
 *
 * The data and maxlen fields of the ctl_table
 * struct enable minimal validation of the values being written to be
 * performed, and the mode field allows minimal authentication.
 * 
 * There must be a proc_handler routine for any terminal nodes
 * mirrored under /proc/sys (non-terminals are handled by a built-in
 * directory handler).  Several default handlers are available to
 * cover common cases.
 */

/* Support for userspace poll() to watch for changes */
struct ctl_table_poll {
        atomic_t event;
        wait_queue_head_t wait;
};

static inline void *proc_sys_poll_event(struct ctl_table_poll *poll)
{
        return (void *)(unsigned long)atomic_read(&poll->event);
}

#define __CTL_TABLE_POLL_INITIALIZER(name) {                                \
        .event = ATOMIC_INIT(0),                                        \
        .wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.wait) }

#define DEFINE_CTL_TABLE_POLL(name)                                        \
        struct ctl_table_poll name = __CTL_TABLE_POLL_INITIALIZER(name)

/* A sysctl table is an array of struct ctl_table: */
struct ctl_table {
        const char *procname;                /* Text ID for /proc/sys */
        void *data;
        int maxlen;
        umode_t mode;
        proc_handler *proc_handler;        /* Callback for text formatting */
        struct ctl_table_poll *poll;
        void *extra1;
        void *extra2;
} __randomize_layout;

struct ctl_node {
        struct rb_node node;
        struct ctl_table_header *header;
};

/**
 * struct ctl_table_header - maintains dynamic lists of struct ctl_table trees
 * @ctl_table: pointer to the first element in ctl_table array
 * @ctl_table_size: number of elements pointed by @ctl_table
 * @used: The entry will never be touched when equal to 0.
 * @count: Upped every time something is added to @inodes and downed every time
 *         something is removed from inodes
 * @nreg: When nreg drops to 0 the ctl_table_header will be unregistered.
 * @rcu: Delays the freeing of the inode. Introduced with "unfuck proc_sysctl ->d_compare()"
 *
 */
struct ctl_table_header {
        union {
                struct {
                        const struct ctl_table *ctl_table;
                        int ctl_table_size;
                        int used;
                        int count;
                        int nreg;
                };
                struct rcu_head rcu;
        };
        struct completion *unregistering;
        const struct ctl_table *ctl_table_arg;
        struct ctl_table_root *root;
        struct ctl_table_set *set;
        struct ctl_dir *parent;
        struct ctl_node *node;
        struct hlist_head inodes; /* head for proc_inode->sysctl_inodes */
        /**
         * enum type - Enumeration to differentiate between ctl target types
         * @SYSCTL_TABLE_TYPE_DEFAULT: ctl target with no special considerations
         * @SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY: Used to identify a permanently
         *                                       empty directory target to serve
         *                                       as mount point.
         */
        enum {
                SYSCTL_TABLE_TYPE_DEFAULT,
                SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY,
        } type;
};

struct ctl_dir {
        /* Header must be at the start of ctl_dir */
        struct ctl_table_header header;
        struct rb_root root;
};

struct ctl_table_set {
        int (*is_seen)(struct ctl_table_set *);
        struct ctl_dir dir;
};

struct ctl_table_root {
        struct ctl_table_set default_set;
        struct ctl_table_set *(*lookup)(struct ctl_table_root *root);
        void (*set_ownership)(struct ctl_table_header *head,
                              kuid_t *uid, kgid_t *gid);
        int (*permissions)(struct ctl_table_header *head, const struct ctl_table *table);
};

#define register_sysctl(path, table)        \
        register_sysctl_sz(path, table, ARRAY_SIZE(table))

#ifdef CONFIG_SYSCTL

void proc_sys_poll_notify(struct ctl_table_poll *poll);

extern void setup_sysctl_set(struct ctl_table_set *p,
        struct ctl_table_root *root,
        int (*is_seen)(struct ctl_table_set *));
extern void retire_sysctl_set(struct ctl_table_set *set);

struct ctl_table_header *__register_sysctl_table(
        struct ctl_table_set *set,
        const char *path, const struct ctl_table *table, size_t table_size);
struct ctl_table_header *register_sysctl_sz(const char *path, const struct ctl_table *table,
                                            size_t table_size);
void unregister_sysctl_table(struct ctl_table_header * table);

extern int sysctl_init_bases(void);
extern void __register_sysctl_init(const char *path, const struct ctl_table *table,
                                 const char *table_name, size_t table_size);
#define register_sysctl_init(path, table)        \
        __register_sysctl_init(path, table, #table, ARRAY_SIZE(table))
extern struct ctl_table_header *register_sysctl_mount_point(const char *path);

void do_sysctl_args(void);
bool sysctl_is_alias(char *param);
int do_proc_douintvec(const struct ctl_table *table, int write,
                      void *buffer, size_t *lenp, loff_t *ppos,
                      int (*conv)(unsigned long *lvalp,
                                  unsigned int *valp,
                                  int write, void *data),
                      void *data);

extern int pwrsw_enabled;
extern int unaligned_enabled;
extern int unaligned_dump_stack;
extern int no_unaligned_warning;

#else /* CONFIG_SYSCTL */

static inline void register_sysctl_init(const char *path, const struct ctl_table *table)
{
}

static inline struct ctl_table_header *register_sysctl_mount_point(const char *path)
{
        return NULL;
}

static inline struct ctl_table_header *register_sysctl_sz(const char *path,
                                                          const struct ctl_table *table,
                                                          size_t table_size)
{
        return NULL;
}

static inline void unregister_sysctl_table(struct ctl_table_header * table)
{
}

static inline void setup_sysctl_set(struct ctl_table_set *p,
        struct ctl_table_root *root,
        int (*is_seen)(struct ctl_table_set *))
{
}

static inline void do_sysctl_args(void)
{
}

static inline bool sysctl_is_alias(char *param)
{
        return false;
}
#endif /* CONFIG_SYSCTL */

int sysctl_max_threads(const struct ctl_table *table, int write, void *buffer,
                size_t *lenp, loff_t *ppos);

#endif /* _LINUX_SYSCTL_H */








































































































































































































































































































































































































































































































































































































































































































    1 



    1 
    1 




























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/fs/msdos/namei.c
 *
 *  Written 1992,1993 by Werner Almesberger
 *  Hidden files 1995 by Albert Cahalan <albert@ccs.neu.edu> <adc@coe.neu.edu>
 *  Rewritten for constant inumbers 1999 by Al Viro
 */

#include <linux/module.h>
#include <linux/iversion.h>
#include "fat.h"

/* Characters that are undesirable in an MS-DOS file name */
static unsigned char bad_chars[] = "*?<>|\"";
static unsigned char bad_if_strict[] = "+=,; ";

/***** Formats an MS-DOS file name. Rejects invalid names. */
static int msdos_format_name(const unsigned char *name, int len,
                             unsigned char *res, struct fat_mount_options *opts)
        /*
         * name is the proposed name, len is its length, res is
         * the resulting name, opts->name_check is either (r)elaxed,
         * (n)ormal or (s)trict, opts->dotsOK allows dots at the
         * beginning of name (for hidden files)
         */
{
        unsigned char *walk;
        unsigned char c;
        int space;

        if (name[0] == '.') {        /* dotfile because . and .. already done */
                if (opts->dotsOK) {
                        /* Get rid of dot - test for it elsewhere */
                        name++;
                        len--;
                } else
                        return -EINVAL;
        }
        /*
         * disallow names that _really_ start with a dot
         */
        space = 1;
        c = 0;
        for (walk = res; len && walk - res < 8; walk++) {
                c = *name++;
                len--;
                if (opts->name_check != 'r' && strchr(bad_chars, c))
                        return -EINVAL;
                if (opts->name_check == 's' && strchr(bad_if_strict, c))
                        return -EINVAL;
                if (c >= 'A' && c <= 'Z' && opts->name_check == 's')
                        return -EINVAL;
                if (c < ' ' || c == ':' || c == '\\')
                        return -EINVAL;
        /*
         * 0xE5 is legal as a first character, but we must substitute
         * 0x05 because 0xE5 marks deleted files.  Yes, DOS really
         * does this.
         * It seems that Microsoft hacked DOS to support non-US
         * characters after the 0xE5 character was already in use to
         * mark deleted files.
         */
                if ((res == walk) && (c == 0xE5))
                        c = 0x05;
                if (c == '.')
                        break;
                space = (c == ' ');
                *walk = (!opts->nocase && c >= 'a' && c <= 'z') ? c - 32 : c;
        }
        if (space)
                return -EINVAL;
        if (opts->name_check == 's' && len && c != '.') {
                c = *name++;
                len--;
                if (c != '.')
                        return -EINVAL;
        }
        while (c != '.' && len--)
                c = *name++;
        if (c == '.') {
                while (walk - res < 8)
                        *walk++ = ' ';
                while (len > 0 && walk - res < MSDOS_NAME) {
                        c = *name++;
                        len--;
                        if (opts->name_check != 'r' && strchr(bad_chars, c))
                                return -EINVAL;
                        if (opts->name_check == 's' &&
                            strchr(bad_if_strict, c))
                                return -EINVAL;
                        if (c < ' ' || c == ':' || c == '\\')
                                return -EINVAL;
                        if (c == '.') {
                                if (opts->name_check == 's')
                                        return -EINVAL;
                                break;
                        }
                        if (c >= 'A' && c <= 'Z' && opts->name_check == 's')
                                return -EINVAL;
                        space = c == ' ';
                        if (!opts->nocase && c >= 'a' && c <= 'z')
                                *walk++ = c - 32;
                        else
                                *walk++ = c;
                }
                if (space)
                        return -EINVAL;
                if (opts->name_check == 's' && len)
                        return -EINVAL;
        }
        while (walk - res < MSDOS_NAME)
                *walk++ = ' ';

        return 0;
}

/***** Locates a directory entry.  Uses unformatted name. */
static int msdos_find(struct inode *dir, const unsigned char *name, int len,
                      struct fat_slot_info *sinfo)
{
        struct msdos_sb_info *sbi = MSDOS_SB(dir->i_sb);
        unsigned char msdos_name[MSDOS_NAME];
        int err;

        err = msdos_format_name(name, len, msdos_name, &sbi->options);
        if (err)
                return -ENOENT;

        err = fat_scan(dir, msdos_name, sinfo);
        if (!err && sbi->options.dotsOK) {
                if (name[0] == '.') {
                        if (!(sinfo->de->attr & ATTR_HIDDEN))
                                err = -ENOENT;
                } else {
                        if (sinfo->de->attr & ATTR_HIDDEN)
                                err = -ENOENT;
                }
                if (err)
                        brelse(sinfo->bh);
        }
        return err;
}

/*
 * Compute the hash for the msdos name corresponding to the dentry.
 * Note: if the name is invalid, we leave the hash code unchanged so
 * that the existing dentry can be used. The msdos fs routines will
 * return ENOENT or EINVAL as appropriate.
 */
static int msdos_hash(const struct dentry *dentry, struct qstr *qstr)
{
        struct fat_mount_options *options = &MSDOS_SB(dentry->d_sb)->options;
        unsigned char msdos_name[MSDOS_NAME];
        int error;

        error = msdos_format_name(qstr->name, qstr->len, msdos_name, options);
        if (!error)
                qstr->hash = full_name_hash(dentry, msdos_name, MSDOS_NAME);
        return 0;
}

/*
 * Compare two msdos names. If either of the names are invalid,
 * we fall back to doing the standard name comparison.
 */
static int msdos_cmp(const struct dentry *dentry,
                unsigned int len, const char *str, const struct qstr *name)
{
        struct fat_mount_options *options = &MSDOS_SB(dentry->d_sb)->options;
        unsigned char a_msdos_name[MSDOS_NAME], b_msdos_name[MSDOS_NAME];
        int error;

        error = msdos_format_name(name->name, name->len, a_msdos_name, options);
        if (error)
                goto old_compare;
        error = msdos_format_name(str, len, b_msdos_name, options);
        if (error)
                goto old_compare;
        error = memcmp(a_msdos_name, b_msdos_name, MSDOS_NAME);
out:
        return error;

old_compare:
        error = 1;
        if (name->len == len)
                error = memcmp(name->name, str, len);
        goto out;
}

static const struct dentry_operations msdos_dentry_operations = {
        .d_hash                = msdos_hash,
        .d_compare        = msdos_cmp,
};

/*
 * AV. Wrappers for FAT sb operations. Is it wise?
 */

/***** Get inode using directory and name */
static struct dentry *msdos_lookup(struct inode *dir, struct dentry *dentry,
                                   unsigned int flags)
{
        struct super_block *sb = dir->i_sb;
        struct fat_slot_info sinfo;
        struct inode *inode;
        int err;

        mutex_lock(&MSDOS_SB(sb)->s_lock);
        err = msdos_find(dir, dentry->d_name.name, dentry->d_name.len, &sinfo);
        switch (err) {
        case -ENOENT:
                inode = NULL;
                break;
        case 0:
                inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos);
                brelse(sinfo.bh);
                break;
        default:
                inode = ERR_PTR(err);
        }
        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        return d_splice_alias(inode, dentry);
}

/***** Creates a directory entry (name is already formatted). */
static int msdos_add_entry(struct inode *dir, const unsigned char *name,
                           int is_dir, int is_hid, int cluster,
                           struct timespec64 *ts, struct fat_slot_info *sinfo)
{
        struct msdos_sb_info *sbi = MSDOS_SB(dir->i_sb);
        struct msdos_dir_entry de;
        __le16 time, date;
        int err;

        memcpy(de.name, name, MSDOS_NAME);
        de.attr = is_dir ? ATTR_DIR : ATTR_ARCH;
        if (is_hid)
                de.attr |= ATTR_HIDDEN;
        de.lcase = 0;
        fat_time_unix2fat(sbi, ts, &time, &date, NULL);
        de.cdate = de.adate = 0;
        de.ctime = 0;
        de.ctime_cs = 0;
        de.time = time;
        de.date = date;
        fat_set_start(&de, cluster);
        de.size = 0;

        err = fat_add_entries(dir, &de, 1, sinfo);
        if (err)
                return err;

        fat_truncate_time(dir, ts, S_CTIME|S_MTIME);
        if (IS_DIRSYNC(dir))
                (void)fat_sync_inode(dir);
        else
                mark_inode_dirty(dir);

        return 0;
}

/***** Create a file */
static int msdos_create(struct mnt_idmap *idmap, struct inode *dir,
                        struct dentry *dentry, umode_t mode, bool excl)
{
        struct super_block *sb = dir->i_sb;
        struct inode *inode = NULL;
        struct fat_slot_info sinfo;
        struct timespec64 ts;
        unsigned char msdos_name[MSDOS_NAME];
        int err, is_hid;

        mutex_lock(&MSDOS_SB(sb)->s_lock);

        err = msdos_format_name(dentry->d_name.name, dentry->d_name.len,
                                msdos_name, &MSDOS_SB(sb)->options);
        if (err)
                goto out;
        is_hid = (dentry->d_name.name[0] == '.') && (msdos_name[0] != '.');
        /* Have to do it due to foo vs. .foo conflicts */
        if (!fat_scan(dir, msdos_name, &sinfo)) {
                brelse(sinfo.bh);
                err = -EINVAL;
                goto out;
        }

        ts = current_time(dir);
        err = msdos_add_entry(dir, msdos_name, 0, is_hid, 0, &ts, &sinfo);
        if (err)
                goto out;
        inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos);
        brelse(sinfo.bh);
        if (IS_ERR(inode)) {
                err = PTR_ERR(inode);
                goto out;
        }
        fat_truncate_time(inode, &ts, S_ATIME|S_CTIME|S_MTIME);
        /* timestamp is already written, so mark_inode_dirty() is unneeded. */

        d_instantiate(dentry, inode);
out:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        if (!err)
                err = fat_flush_inodes(sb, dir, inode);
        return err;
}

/***** Remove a directory */
static int msdos_rmdir(struct inode *dir, struct dentry *dentry)
{
        struct super_block *sb = dir->i_sb;
        struct inode *inode = d_inode(dentry);
        struct fat_slot_info sinfo;
        int err;

        mutex_lock(&MSDOS_SB(sb)->s_lock);
        err = fat_dir_empty(inode);
        if (err)
                goto out;
        err = msdos_find(dir, dentry->d_name.name, dentry->d_name.len, &sinfo);
        if (err)
                goto out;

        err = fat_remove_entries(dir, &sinfo);        /* and releases bh */
        if (err)
                goto out;
        drop_nlink(dir);

        clear_nlink(inode);
        fat_truncate_time(inode, NULL, S_CTIME);
        fat_detach(inode);
out:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        if (!err)
                err = fat_flush_inodes(sb, dir, inode);

        return err;
}

/***** Make a directory */
static int msdos_mkdir(struct mnt_idmap *idmap, struct inode *dir,
                       struct dentry *dentry, umode_t mode)
{
        struct super_block *sb = dir->i_sb;
        struct fat_slot_info sinfo;
        struct inode *inode;
        unsigned char msdos_name[MSDOS_NAME];
        struct timespec64 ts;
        int err, is_hid, cluster;

        mutex_lock(&MSDOS_SB(sb)->s_lock);

        err = msdos_format_name(dentry->d_name.name, dentry->d_name.len,
                                msdos_name, &MSDOS_SB(sb)->options);
        if (err)
                goto out;
        is_hid = (dentry->d_name.name[0] == '.') && (msdos_name[0] != '.');
        /* foo vs .foo situation */
        if (!fat_scan(dir, msdos_name, &sinfo)) {
                brelse(sinfo.bh);
                err = -EINVAL;
                goto out;
        }

        ts = current_time(dir);
        cluster = fat_alloc_new_dir(dir, &ts);
        if (cluster < 0) {
                err = cluster;
                goto out;
        }
        err = msdos_add_entry(dir, msdos_name, 1, is_hid, cluster, &ts, &sinfo);
        if (err)
                goto out_free;
        inc_nlink(dir);

        inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos);
        brelse(sinfo.bh);
        if (IS_ERR(inode)) {
                err = PTR_ERR(inode);
                /* the directory was completed, just return a error */
                goto out;
        }
        set_nlink(inode, 2);
        fat_truncate_time(inode, &ts, S_ATIME|S_CTIME|S_MTIME);
        /* timestamp is already written, so mark_inode_dirty() is unneeded. */

        d_instantiate(dentry, inode);

        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        fat_flush_inodes(sb, dir, inode);
        return 0;

out_free:
        fat_free_clusters(dir, cluster);
out:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        return err;
}

/***** Unlink a file */
static int msdos_unlink(struct inode *dir, struct dentry *dentry)
{
        struct inode *inode = d_inode(dentry);
        struct super_block *sb = inode->i_sb;
        struct fat_slot_info sinfo;
        int err;

        mutex_lock(&MSDOS_SB(sb)->s_lock);
        err = msdos_find(dir, dentry->d_name.name, dentry->d_name.len, &sinfo);
        if (err)
                goto out;

        err = fat_remove_entries(dir, &sinfo);        /* and releases bh */
        if (err)
                goto out;
        clear_nlink(inode);
        fat_truncate_time(inode, NULL, S_CTIME);
        fat_detach(inode);
out:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        if (!err)
                err = fat_flush_inodes(sb, dir, inode);

        return err;
}

static int do_msdos_rename(struct inode *old_dir, unsigned char *old_name,
                           struct dentry *old_dentry,
                           struct inode *new_dir, unsigned char *new_name,
                           struct dentry *new_dentry, int is_hid)
{
        struct buffer_head *dotdot_bh;
        struct msdos_dir_entry *dotdot_de;
        struct inode *old_inode, *new_inode;
        struct fat_slot_info old_sinfo, sinfo;
        struct timespec64 ts;
        loff_t new_i_pos;
        int err, old_attrs, is_dir, update_dotdot, corrupt = 0;

        old_sinfo.bh = sinfo.bh = dotdot_bh = NULL;
        old_inode = d_inode(old_dentry);
        new_inode = d_inode(new_dentry);

        err = fat_scan(old_dir, old_name, &old_sinfo);
        if (err) {
                err = -EIO;
                goto out;
        }

        is_dir = S_ISDIR(old_inode->i_mode);
        update_dotdot = (is_dir && old_dir != new_dir);
        if (update_dotdot) {
                if (fat_get_dotdot_entry(old_inode, &dotdot_bh, &dotdot_de)) {
                        err = -EIO;
                        goto out;
                }
        }

        old_attrs = MSDOS_I(old_inode)->i_attrs;
        err = fat_scan(new_dir, new_name, &sinfo);
        if (!err) {
                if (!new_inode) {
                        /* "foo" -> ".foo" case. just change the ATTR_HIDDEN */
                        if (sinfo.de != old_sinfo.de) {
                                err = -EINVAL;
                                goto out;
                        }
                        if (is_hid)
                                MSDOS_I(old_inode)->i_attrs |= ATTR_HIDDEN;
                        else
                                MSDOS_I(old_inode)->i_attrs &= ~ATTR_HIDDEN;
                        if (IS_DIRSYNC(old_dir)) {
                                err = fat_sync_inode(old_inode);
                                if (err) {
                                        MSDOS_I(old_inode)->i_attrs = old_attrs;
                                        goto out;
                                }
                        } else
                                mark_inode_dirty(old_inode);

                        inode_inc_iversion(old_dir);
                        fat_truncate_time(old_dir, NULL, S_CTIME|S_MTIME);
                        if (IS_DIRSYNC(old_dir))
                                (void)fat_sync_inode(old_dir);
                        else
                                mark_inode_dirty(old_dir);
                        goto out;
                }
        }

        ts = current_time(old_inode);
        if (new_inode) {
                if (err)
                        goto out;
                if (is_dir) {
                        err = fat_dir_empty(new_inode);
                        if (err)
                                goto out;
                }
                new_i_pos = MSDOS_I(new_inode)->i_pos;
                fat_detach(new_inode);
        } else {
                err = msdos_add_entry(new_dir, new_name, is_dir, is_hid, 0,
                                      &ts, &sinfo);
                if (err)
                        goto out;
                new_i_pos = sinfo.i_pos;
        }
        inode_inc_iversion(new_dir);

        fat_detach(old_inode);
        fat_attach(old_inode, new_i_pos);
        if (is_hid)
                MSDOS_I(old_inode)->i_attrs |= ATTR_HIDDEN;
        else
                MSDOS_I(old_inode)->i_attrs &= ~ATTR_HIDDEN;
        if (IS_DIRSYNC(new_dir)) {
                err = fat_sync_inode(old_inode);
                if (err)
                        goto error_inode;
        } else
                mark_inode_dirty(old_inode);

        if (update_dotdot) {
                fat_set_start(dotdot_de, MSDOS_I(new_dir)->i_logstart);
                mark_buffer_dirty_inode(dotdot_bh, old_inode);
                if (IS_DIRSYNC(new_dir)) {
                        err = sync_dirty_buffer(dotdot_bh);
                        if (err)
                                goto error_dotdot;
                }
                drop_nlink(old_dir);
                if (!new_inode)
                        inc_nlink(new_dir);
        }

        err = fat_remove_entries(old_dir, &old_sinfo);        /* and releases bh */
        old_sinfo.bh = NULL;
        if (err)
                goto error_dotdot;
        inode_inc_iversion(old_dir);
        fat_truncate_time(old_dir, &ts, S_CTIME|S_MTIME);
        if (IS_DIRSYNC(old_dir))
                (void)fat_sync_inode(old_dir);
        else
                mark_inode_dirty(old_dir);

        if (new_inode) {
                drop_nlink(new_inode);
                if (is_dir)
                        drop_nlink(new_inode);
                fat_truncate_time(new_inode, &ts, S_CTIME);
        }
out:
        brelse(sinfo.bh);
        brelse(dotdot_bh);
        brelse(old_sinfo.bh);
        return err;

error_dotdot:
        /* data cluster is shared, serious corruption */
        corrupt = 1;

        if (update_dotdot) {
                fat_set_start(dotdot_de, MSDOS_I(old_dir)->i_logstart);
                mark_buffer_dirty_inode(dotdot_bh, old_inode);
                corrupt |= sync_dirty_buffer(dotdot_bh);
        }
error_inode:
        fat_detach(old_inode);
        fat_attach(old_inode, old_sinfo.i_pos);
        MSDOS_I(old_inode)->i_attrs = old_attrs;
        if (new_inode) {
                fat_attach(new_inode, new_i_pos);
                if (corrupt)
                        corrupt |= fat_sync_inode(new_inode);
        } else {
                /*
                 * If new entry was not sharing the data cluster, it
                 * shouldn't be serious corruption.
                 */
                int err2 = fat_remove_entries(new_dir, &sinfo);
                if (corrupt)
                        corrupt |= err2;
                sinfo.bh = NULL;
        }
        if (corrupt < 0) {
                fat_fs_error(new_dir->i_sb,
                             "%s: Filesystem corrupted (i_pos %lld)",
                             __func__, sinfo.i_pos);
        }
        goto out;
}

/***** Rename, a wrapper for rename_same_dir & rename_diff_dir */
static int msdos_rename(struct mnt_idmap *idmap,
                        struct inode *old_dir, struct dentry *old_dentry,
                        struct inode *new_dir, struct dentry *new_dentry,
                        unsigned int flags)
{
        struct super_block *sb = old_dir->i_sb;
        unsigned char old_msdos_name[MSDOS_NAME], new_msdos_name[MSDOS_NAME];
        int err, is_hid;

        if (flags & ~RENAME_NOREPLACE)
                return -EINVAL;

        mutex_lock(&MSDOS_SB(sb)->s_lock);

        err = msdos_format_name(old_dentry->d_name.name,
                                old_dentry->d_name.len, old_msdos_name,
                                &MSDOS_SB(old_dir->i_sb)->options);
        if (err)
                goto out;
        err = msdos_format_name(new_dentry->d_name.name,
                                new_dentry->d_name.len, new_msdos_name,
                                &MSDOS_SB(new_dir->i_sb)->options);
        if (err)
                goto out;

        is_hid =
             (new_dentry->d_name.name[0] == '.') && (new_msdos_name[0] != '.');

        err = do_msdos_rename(old_dir, old_msdos_name, old_dentry,
                              new_dir, new_msdos_name, new_dentry, is_hid);
out:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        if (!err)
                err = fat_flush_inodes(sb, old_dir, new_dir);
        return err;
}

static const struct inode_operations msdos_dir_inode_operations = {
        .create                = msdos_create,
        .lookup                = msdos_lookup,
        .unlink                = msdos_unlink,
        .mkdir                = msdos_mkdir,
        .rmdir                = msdos_rmdir,
        .rename                = msdos_rename,
        .setattr        = fat_setattr,
        .getattr        = fat_getattr,
        .update_time        = fat_update_time,
};

static void setup(struct super_block *sb)
{
        MSDOS_SB(sb)->dir_ops = &msdos_dir_inode_operations;
        sb->s_d_op = &msdos_dentry_operations;
        sb->s_flags |= SB_NOATIME;
}

static int msdos_fill_super(struct super_block *sb, struct fs_context *fc)
{
        return fat_fill_super(sb, fc, setup);
}

static int msdos_get_tree(struct fs_context *fc)
{
        return get_tree_bdev(fc, msdos_fill_super);
}

static int msdos_parse_param(struct fs_context *fc, struct fs_parameter *param)
{
        return fat_parse_param(fc, param, false);
}

static const struct fs_context_operations msdos_context_ops = {
        .parse_param        = msdos_parse_param,
        .get_tree        = msdos_get_tree,
        .reconfigure        = fat_reconfigure,
        .free                = fat_free_fc,
};

static int msdos_init_fs_context(struct fs_context *fc)
{
        int err;

        /* Initialize with is_vfat == false */
        err = fat_init_fs_context(fc, false);
        if (err)
                return err;

        fc->ops = &msdos_context_ops;
        return 0;
}

static struct file_system_type msdos_fs_type = {
        .owner                = THIS_MODULE,
        .name                = "msdos",
        .kill_sb        = kill_block_super,
        .fs_flags        = FS_REQUIRES_DEV | FS_ALLOW_IDMAP,
        .init_fs_context = msdos_init_fs_context,
        .parameters        = fat_param_spec,
};
MODULE_ALIAS_FS("msdos");

static int __init init_msdos_fs(void)
{
        return register_filesystem(&msdos_fs_type);
}

static void __exit exit_msdos_fs(void)
{
        unregister_filesystem(&msdos_fs_type);
}

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Werner Almesberger");
MODULE_DESCRIPTION("MS-DOS filesystem support");

module_init(init_msdos_fs)
module_exit(exit_msdos_fs)













































































   48 













   48 


   48 











    8 




    8 



   40 



   41 



   46 




   14 












   14 






   14 
   14 

   37 
    8 


   35 


   45 


   42 

    6 
   42 
    6 

   41 

   45 




    8 

    8 










    8 


















    9 





    6 












   21 
    3 

   21 






   20 













   20 



   20 


















   10 
    2 

   10 






   10 













    9 



    9 





   29 















   36 




   11 








   25 



   35 



    6 

   35 














   21 



   23 





   23 








   40 
    1 











   24 
    1 































    9 














    2 




    2 











   10 
    3 




    3 

    1 












    9 
    2 





    2 













    8 
    6 






    6 

    7 












   10 


    9 

















   10 








   10 




















    9 

   10 





   10 































  118 











  119 




   38 
   21 


   20 













   13 
   10 


   29 







   49 
   51 
   51 



   37 


   35 


   33 
   10 

   23 

   33 



   37 

   28 
    3 





   38 





    7 






   10 


   15 








    1 



   38 




    1 















   37 




   19 


   12 


























   98 
   98 
   61 




    8 






   34 
   36 






   61 




   94 



   91 


  120 
  118 


   13 
   13 





   60 






















   14 


   14 



   11 



   11 







    3 



    1 

    3 
















    2 



    2 



    2 






    1 

    1 
   12 





























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
// SPDX-License-Identifier: GPL-2.0-or-later
/* Manage a process's keyrings
 *
 * Copyright (C) 2004-2005, 2008 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/init.h>
#include <linux/sched.h>
#include <linux/sched/user.h>
#include <linux/keyctl.h>
#include <linux/fs.h>
#include <linux/err.h>
#include <linux/mutex.h>
#include <linux/security.h>
#include <linux/user_namespace.h>
#include <linux/uaccess.h>
#include <linux/init_task.h>
#include <keys/request_key_auth-type.h>
#include "internal.h"

/* Session keyring create vs join semaphore */
static DEFINE_MUTEX(key_session_mutex);

/* The root user's tracking struct */
struct key_user root_key_user = {
        .usage                = REFCOUNT_INIT(3),
        .cons_lock        = __MUTEX_INITIALIZER(root_key_user.cons_lock),
        .lock                = __SPIN_LOCK_UNLOCKED(root_key_user.lock),
        .nkeys                = ATOMIC_INIT(2),
        .nikeys                = ATOMIC_INIT(2),
        .uid                = GLOBAL_ROOT_UID,
};

/*
 * Get or create a user register keyring.
 */
static struct key *get_user_register(struct user_namespace *user_ns)
{
        struct key *reg_keyring = READ_ONCE(user_ns->user_keyring_register);

        if (reg_keyring)
                return reg_keyring;

        down_write(&user_ns->keyring_sem);

        /* Make sure there's a register keyring.  It gets owned by the
         * user_namespace's owner.
         */
        reg_keyring = user_ns->user_keyring_register;
        if (!reg_keyring) {
                reg_keyring = keyring_alloc(".user_reg",
                                            user_ns->owner, INVALID_GID,
                                            &init_cred,
                                            KEY_POS_WRITE | KEY_POS_SEARCH |
                                            KEY_USR_VIEW | KEY_USR_READ,
                                            0,
                                            NULL, NULL);
                if (!IS_ERR(reg_keyring))
                        smp_store_release(&user_ns->user_keyring_register,
                                          reg_keyring);
        }

        up_write(&user_ns->keyring_sem);

        /* We don't return a ref since the keyring is pinned by the user_ns */
        return reg_keyring;
}

/*
 * Look up the user and user session keyrings for the current process's UID,
 * creating them if they don't exist.
 */
int look_up_user_keyrings(struct key **_user_keyring,
                          struct key **_user_session_keyring)
{
        const struct cred *cred = current_cred();
        struct user_namespace *user_ns = current_user_ns();
        struct key *reg_keyring, *uid_keyring, *session_keyring;
        key_perm_t user_keyring_perm;
        key_ref_t uid_keyring_r, session_keyring_r;
        uid_t uid = from_kuid(user_ns, cred->user->uid);
        char buf[20];
        int ret;

        user_keyring_perm = (KEY_POS_ALL & ~KEY_POS_SETATTR) | KEY_USR_ALL;

        kenter("%u", uid);

        reg_keyring = get_user_register(user_ns);
        if (IS_ERR(reg_keyring))
                return PTR_ERR(reg_keyring);

        down_write(&user_ns->keyring_sem);
        ret = 0;

        /* Get the user keyring.  Note that there may be one in existence
         * already as it may have been pinned by a session, but the user_struct
         * pointing to it may have been destroyed by setuid.
         */
        snprintf(buf, sizeof(buf), "_uid.%u", uid);
        uid_keyring_r = keyring_search(make_key_ref(reg_keyring, true),
                                       &key_type_keyring, buf, false);
        kdebug("_uid %p", uid_keyring_r);
        if (uid_keyring_r == ERR_PTR(-EAGAIN)) {
                uid_keyring = keyring_alloc(buf, cred->user->uid, INVALID_GID,
                                            cred, user_keyring_perm,
                                            KEY_ALLOC_UID_KEYRING |
                                            KEY_ALLOC_IN_QUOTA,
                                            NULL, reg_keyring);
                if (IS_ERR(uid_keyring)) {
                        ret = PTR_ERR(uid_keyring);
                        goto error;
                }
        } else if (IS_ERR(uid_keyring_r)) {
                ret = PTR_ERR(uid_keyring_r);
                goto error;
        } else {
                uid_keyring = key_ref_to_ptr(uid_keyring_r);
        }

        /* Get a default session keyring (which might also exist already) */
        snprintf(buf, sizeof(buf), "_uid_ses.%u", uid);
        session_keyring_r = keyring_search(make_key_ref(reg_keyring, true),
                                           &key_type_keyring, buf, false);
        kdebug("_uid_ses %p", session_keyring_r);
        if (session_keyring_r == ERR_PTR(-EAGAIN)) {
                session_keyring = keyring_alloc(buf, cred->user->uid, INVALID_GID,
                                                cred, user_keyring_perm,
                                                KEY_ALLOC_UID_KEYRING |
                                                KEY_ALLOC_IN_QUOTA,
                                                NULL, NULL);
                if (IS_ERR(session_keyring)) {
                        ret = PTR_ERR(session_keyring);
                        goto error_release;
                }

                /* We install a link from the user session keyring to
                 * the user keyring.
                 */
                ret = key_link(session_keyring, uid_keyring);
                if (ret < 0)
                        goto error_release_session;

                /* And only then link the user-session keyring to the
                 * register.
                 */
                ret = key_link(reg_keyring, session_keyring);
                if (ret < 0)
                        goto error_release_session;
        } else if (IS_ERR(session_keyring_r)) {
                ret = PTR_ERR(session_keyring_r);
                goto error_release;
        } else {
                session_keyring = key_ref_to_ptr(session_keyring_r);
        }

        up_write(&user_ns->keyring_sem);

        if (_user_session_keyring)
                *_user_session_keyring = session_keyring;
        else
                key_put(session_keyring);
        if (_user_keyring)
                *_user_keyring = uid_keyring;
        else
                key_put(uid_keyring);
        kleave(" = 0");
        return 0;

error_release_session:
        key_put(session_keyring);
error_release:
        key_put(uid_keyring);
error:
        up_write(&user_ns->keyring_sem);
        kleave(" = %d", ret);
        return ret;
}

/*
 * Get the user session keyring if it exists, but don't create it if it
 * doesn't.
 */
struct key *get_user_session_keyring_rcu(const struct cred *cred)
{
        struct key *reg_keyring = READ_ONCE(cred->user_ns->user_keyring_register);
        key_ref_t session_keyring_r;
        char buf[20];

        struct keyring_search_context ctx = {
                .index_key.type                = &key_type_keyring,
                .index_key.description        = buf,
                .cred                        = cred,
                .match_data.cmp                = key_default_cmp,
                .match_data.raw_data        = buf,
                .match_data.lookup_type        = KEYRING_SEARCH_LOOKUP_DIRECT,
                .flags                        = KEYRING_SEARCH_DO_STATE_CHECK,
        };

        if (!reg_keyring)
                return NULL;

        ctx.index_key.desc_len = snprintf(buf, sizeof(buf), "_uid_ses.%u",
                                          from_kuid(cred->user_ns,
                                                    cred->user->uid));

        session_keyring_r = keyring_search_rcu(make_key_ref(reg_keyring, true),
                                               &ctx);
        if (IS_ERR(session_keyring_r))
                return NULL;
        return key_ref_to_ptr(session_keyring_r);
}

/*
 * Install a thread keyring to the given credentials struct if it didn't have
 * one already.  This is allowed to overrun the quota.
 *
 * Return: 0 if a thread keyring is now present; -errno on failure.
 */
int install_thread_keyring_to_cred(struct cred *new)
{
        struct key *keyring;

        if (new->thread_keyring)
                return 0;

        keyring = keyring_alloc("_tid", new->uid, new->gid, new,
                                KEY_POS_ALL | KEY_USR_VIEW,
                                KEY_ALLOC_QUOTA_OVERRUN,
                                NULL, NULL);
        if (IS_ERR(keyring))
                return PTR_ERR(keyring);

        new->thread_keyring = keyring;
        return 0;
}

/*
 * Install a thread keyring to the current task if it didn't have one already.
 *
 * Return: 0 if a thread keyring is now present; -errno on failure.
 */
static int install_thread_keyring(void)
{
        struct cred *new;
        int ret;

        new = prepare_creds();
        if (!new)
                return -ENOMEM;

        ret = install_thread_keyring_to_cred(new);
        if (ret < 0) {
                abort_creds(new);
                return ret;
        }

        return commit_creds(new);
}

/*
 * Install a process keyring to the given credentials struct if it didn't have
 * one already.  This is allowed to overrun the quota.
 *
 * Return: 0 if a process keyring is now present; -errno on failure.
 */
int install_process_keyring_to_cred(struct cred *new)
{
        struct key *keyring;

        if (new->process_keyring)
                return 0;

        keyring = keyring_alloc("_pid", new->uid, new->gid, new,
                                KEY_POS_ALL | KEY_USR_VIEW,
                                KEY_ALLOC_QUOTA_OVERRUN,
                                NULL, NULL);
        if (IS_ERR(keyring))
                return PTR_ERR(keyring);

        new->process_keyring = keyring;
        return 0;
}

/*
 * Install a process keyring to the current task if it didn't have one already.
 *
 * Return: 0 if a process keyring is now present; -errno on failure.
 */
static int install_process_keyring(void)
{
        struct cred *new;
        int ret;

        new = prepare_creds();
        if (!new)
                return -ENOMEM;

        ret = install_process_keyring_to_cred(new);
        if (ret < 0) {
                abort_creds(new);
                return ret;
        }

        return commit_creds(new);
}

/*
 * Install the given keyring as the session keyring of the given credentials
 * struct, replacing the existing one if any.  If the given keyring is NULL,
 * then install a new anonymous session keyring.
 * @cred can not be in use by any task yet.
 *
 * Return: 0 on success; -errno on failure.
 */
int install_session_keyring_to_cred(struct cred *cred, struct key *keyring)
{
        unsigned long flags;
        struct key *old;

        might_sleep();

        /* create an empty session keyring */
        if (!keyring) {
                flags = KEY_ALLOC_QUOTA_OVERRUN;
                if (cred->session_keyring)
                        flags = KEY_ALLOC_IN_QUOTA;

                keyring = keyring_alloc("_ses", cred->uid, cred->gid, cred,
                                        KEY_POS_ALL | KEY_USR_VIEW | KEY_USR_READ,
                                        flags, NULL, NULL);
                if (IS_ERR(keyring))
                        return PTR_ERR(keyring);
        } else {
                __key_get(keyring);
        }

        /* install the keyring */
        old = cred->session_keyring;
        cred->session_keyring = keyring;

        if (old)
                key_put(old);

        return 0;
}

/*
 * Install the given keyring as the session keyring of the current task,
 * replacing the existing one if any.  If the given keyring is NULL, then
 * install a new anonymous session keyring.
 *
 * Return: 0 on success; -errno on failure.
 */
static int install_session_keyring(struct key *keyring)
{
        struct cred *new;
        int ret;

        new = prepare_creds();
        if (!new)
                return -ENOMEM;

        ret = install_session_keyring_to_cred(new, keyring);
        if (ret < 0) {
                abort_creds(new);
                return ret;
        }

        return commit_creds(new);
}

/*
 * Handle the fsuid changing.
 */
void key_fsuid_changed(struct cred *new_cred)
{
        /* update the ownership of the thread keyring */
        if (new_cred->thread_keyring) {
                down_write(&new_cred->thread_keyring->sem);
                new_cred->thread_keyring->uid = new_cred->fsuid;
                up_write(&new_cred->thread_keyring->sem);
        }
}

/*
 * Handle the fsgid changing.
 */
void key_fsgid_changed(struct cred *new_cred)
{
        /* update the ownership of the thread keyring */
        if (new_cred->thread_keyring) {
                down_write(&new_cred->thread_keyring->sem);
                new_cred->thread_keyring->gid = new_cred->fsgid;
                up_write(&new_cred->thread_keyring->sem);
        }
}

/*
 * Search the process keyrings attached to the supplied cred for the first
 * matching key under RCU conditions (the caller must be holding the RCU read
 * lock).
 *
 * The search criteria are the type and the match function.  The description is
 * given to the match function as a parameter, but doesn't otherwise influence
 * the search.  Typically the match function will compare the description
 * parameter to the key's description.
 *
 * This can only search keyrings that grant Search permission to the supplied
 * credentials.  Keyrings linked to searched keyrings will also be searched if
 * they grant Search permission too.  Keys can only be found if they grant
 * Search permission to the credentials.
 *
 * Returns a pointer to the key with the key usage count incremented if
 * successful, -EAGAIN if we didn't find any matching key or -ENOKEY if we only
 * matched negative keys.
 *
 * In the case of a successful return, the possession attribute is set on the
 * returned key reference.
 */
key_ref_t search_cred_keyrings_rcu(struct keyring_search_context *ctx)
{
        struct key *user_session;
        key_ref_t key_ref, ret, err;
        const struct cred *cred = ctx->cred;

        /* we want to return -EAGAIN or -ENOKEY if any of the keyrings were
         * searchable, but we failed to find a key or we found a negative key;
         * otherwise we want to return a sample error (probably -EACCES) if
         * none of the keyrings were searchable
         *
         * in terms of priority: success > -ENOKEY > -EAGAIN > other error
         */
        key_ref = NULL;
        ret = NULL;
        err = ERR_PTR(-EAGAIN);

        /* search the thread keyring first */
        if (cred->thread_keyring) {
                key_ref = keyring_search_rcu(
                        make_key_ref(cred->thread_keyring, 1), ctx);
                if (!IS_ERR(key_ref))
                        goto found;

                switch (PTR_ERR(key_ref)) {
                case -EAGAIN: /* no key */
                case -ENOKEY: /* negative key */
                        ret = key_ref;
                        break;
                default:
                        err = key_ref;
                        break;
                }
        }

        /* search the process keyring second */
        if (cred->process_keyring) {
                key_ref = keyring_search_rcu(
                        make_key_ref(cred->process_keyring, 1), ctx);
                if (!IS_ERR(key_ref))
                        goto found;

                switch (PTR_ERR(key_ref)) {
                case -EAGAIN: /* no key */
                        if (ret)
                                break;
                        fallthrough;
                case -ENOKEY: /* negative key */
                        ret = key_ref;
                        break;
                default:
                        err = key_ref;
                        break;
                }
        }

        /* search the session keyring */
        if (cred->session_keyring) {
                key_ref = keyring_search_rcu(
                        make_key_ref(cred->session_keyring, 1), ctx);

                if (!IS_ERR(key_ref))
                        goto found;

                switch (PTR_ERR(key_ref)) {
                case -EAGAIN: /* no key */
                        if (ret)
                                break;
                        fallthrough;
                case -ENOKEY: /* negative key */
                        ret = key_ref;
                        break;
                default:
                        err = key_ref;
                        break;
                }
        }
        /* or search the user-session keyring */
        else if ((user_session = get_user_session_keyring_rcu(cred))) {
                key_ref = keyring_search_rcu(make_key_ref(user_session, 1),
                                             ctx);
                key_put(user_session);

                if (!IS_ERR(key_ref))
                        goto found;

                switch (PTR_ERR(key_ref)) {
                case -EAGAIN: /* no key */
                        if (ret)
                                break;
                        fallthrough;
                case -ENOKEY: /* negative key */
                        ret = key_ref;
                        break;
                default:
                        err = key_ref;
                        break;
                }
        }

        /* no key - decide on the error we're going to go for */
        key_ref = ret ? ret : err;

found:
        return key_ref;
}

/*
 * Search the process keyrings attached to the supplied cred for the first
 * matching key in the manner of search_my_process_keyrings(), but also search
 * the keys attached to the assumed authorisation key using its credentials if
 * one is available.
 *
 * The caller must be holding the RCU read lock.
 *
 * Return same as search_cred_keyrings_rcu().
 */
key_ref_t search_process_keyrings_rcu(struct keyring_search_context *ctx)
{
        struct request_key_auth *rka;
        key_ref_t key_ref, ret = ERR_PTR(-EACCES), err;

        key_ref = search_cred_keyrings_rcu(ctx);
        if (!IS_ERR(key_ref))
                goto found;
        err = key_ref;

        /* if this process has an instantiation authorisation key, then we also
         * search the keyrings of the process mentioned there
         * - we don't permit access to request_key auth keys via this method
         */
        if (ctx->cred->request_key_auth &&
            ctx->cred == current_cred() &&
            ctx->index_key.type != &key_type_request_key_auth
            ) {
                const struct cred *cred = ctx->cred;

                if (key_validate(cred->request_key_auth) == 0) {
                        rka = ctx->cred->request_key_auth->payload.data[0];

                        //// was search_process_keyrings() [ie. recursive]
                        ctx->cred = rka->cred;
                        key_ref = search_cred_keyrings_rcu(ctx);
                        ctx->cred = cred;

                        if (!IS_ERR(key_ref))
                                goto found;
                        ret = key_ref;
                }
        }

        /* no key - decide on the error we're going to go for */
        if (err == ERR_PTR(-ENOKEY) || ret == ERR_PTR(-ENOKEY))
                key_ref = ERR_PTR(-ENOKEY);
        else if (err == ERR_PTR(-EACCES))
                key_ref = ret;
        else
                key_ref = err;

found:
        return key_ref;
}
/*
 * See if the key we're looking at is the target key.
 */
bool lookup_user_key_possessed(const struct key *key,
                               const struct key_match_data *match_data)
{
        return key == match_data->raw_data;
}

/*
 * Look up a key ID given us by userspace with a given permissions mask to get
 * the key it refers to.
 *
 * Flags can be passed to request that special keyrings be created if referred
 * to directly, to permit partially constructed keys to be found and to skip
 * validity and permission checks on the found key.
 *
 * Returns a pointer to the key with an incremented usage count if successful;
 * -EINVAL if the key ID is invalid; -ENOKEY if the key ID does not correspond
 * to a key or the best found key was a negative key; -EKEYREVOKED or
 * -EKEYEXPIRED if the best found key was revoked or expired; -EACCES if the
 * found key doesn't grant the requested permit or the LSM denied access to it;
 * or -ENOMEM if a special keyring couldn't be created.
 *
 * In the case of a successful return, the possession attribute is set on the
 * returned key reference.
 */
key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags,
                          enum key_need_perm need_perm)
{
        struct keyring_search_context ctx = {
                .match_data.cmp                = lookup_user_key_possessed,
                .match_data.lookup_type        = KEYRING_SEARCH_LOOKUP_DIRECT,
                .flags                        = (KEYRING_SEARCH_NO_STATE_CHECK |
                                           KEYRING_SEARCH_RECURSE),
        };
        struct request_key_auth *rka;
        struct key *key, *user_session;
        key_ref_t key_ref, skey_ref;
        int ret;

try_again:
        ctx.cred = get_current_cred();
        key_ref = ERR_PTR(-ENOKEY);

        switch (id) {
        case KEY_SPEC_THREAD_KEYRING:
                if (!ctx.cred->thread_keyring) {
                        if (!(lflags & KEY_LOOKUP_CREATE))
                                goto error;

                        ret = install_thread_keyring();
                        if (ret < 0) {
                                key_ref = ERR_PTR(ret);
                                goto error;
                        }
                        goto reget_creds;
                }

                key = ctx.cred->thread_keyring;
                __key_get(key);
                key_ref = make_key_ref(key, 1);
                break;

        case KEY_SPEC_PROCESS_KEYRING:
                if (!ctx.cred->process_keyring) {
                        if (!(lflags & KEY_LOOKUP_CREATE))
                                goto error;

                        ret = install_process_keyring();
                        if (ret < 0) {
                                key_ref = ERR_PTR(ret);
                                goto error;
                        }
                        goto reget_creds;
                }

                key = ctx.cred->process_keyring;
                __key_get(key);
                key_ref = make_key_ref(key, 1);
                break;

        case KEY_SPEC_SESSION_KEYRING:
                if (!ctx.cred->session_keyring) {
                        /* always install a session keyring upon access if one
                         * doesn't exist yet */
                        ret = look_up_user_keyrings(NULL, &user_session);
                        if (ret < 0)
                                goto error;
                        if (lflags & KEY_LOOKUP_CREATE)
                                ret = join_session_keyring(NULL);
                        else
                                ret = install_session_keyring(user_session);

                        key_put(user_session);
                        if (ret < 0)
                                goto error;
                        goto reget_creds;
                } else if (test_bit(KEY_FLAG_UID_KEYRING,
                                    &ctx.cred->session_keyring->flags) &&
                           lflags & KEY_LOOKUP_CREATE) {
                        ret = join_session_keyring(NULL);
                        if (ret < 0)
                                goto error;
                        goto reget_creds;
                }

                key = ctx.cred->session_keyring;
                __key_get(key);
                key_ref = make_key_ref(key, 1);
                break;

        case KEY_SPEC_USER_KEYRING:
                ret = look_up_user_keyrings(&key, NULL);
                if (ret < 0)
                        goto error;
                key_ref = make_key_ref(key, 1);
                break;

        case KEY_SPEC_USER_SESSION_KEYRING:
                ret = look_up_user_keyrings(NULL, &key);
                if (ret < 0)
                        goto error;
                key_ref = make_key_ref(key, 1);
                break;

        case KEY_SPEC_GROUP_KEYRING:
                /* group keyrings are not yet supported */
                key_ref = ERR_PTR(-EINVAL);
                goto error;

        case KEY_SPEC_REQKEY_AUTH_KEY:
                key = ctx.cred->request_key_auth;
                if (!key)
                        goto error;

                __key_get(key);
                key_ref = make_key_ref(key, 1);
                break;

        case KEY_SPEC_REQUESTOR_KEYRING:
                if (!ctx.cred->request_key_auth)
                        goto error;

                down_read(&ctx.cred->request_key_auth->sem);
                if (test_bit(KEY_FLAG_REVOKED,
                             &ctx.cred->request_key_auth->flags)) {
                        key_ref = ERR_PTR(-EKEYREVOKED);
                        key = NULL;
                } else {
                        rka = ctx.cred->request_key_auth->payload.data[0];
                        key = rka->dest_keyring;
                        __key_get(key);
                }
                up_read(&ctx.cred->request_key_auth->sem);
                if (!key)
                        goto error;
                key_ref = make_key_ref(key, 1);
                break;

        default:
                key_ref = ERR_PTR(-EINVAL);
                if (id < 1)
                        goto error;

                key = key_lookup(id);
                if (IS_ERR(key)) {
                        key_ref = ERR_CAST(key);
                        goto error;
                }

                key_ref = make_key_ref(key, 0);

                /* check to see if we possess the key */
                ctx.index_key                        = key->index_key;
                ctx.match_data.raw_data                = key;
                kdebug("check possessed");
                rcu_read_lock();
                skey_ref = search_process_keyrings_rcu(&ctx);
                rcu_read_unlock();
                kdebug("possessed=%p", skey_ref);

                if (!IS_ERR(skey_ref)) {
                        key_put(key);
                        key_ref = skey_ref;
                }

                break;
        }

        /* unlink does not use the nominated key in any way, so can skip all
         * the permission checks as it is only concerned with the keyring */
        if (need_perm != KEY_NEED_UNLINK) {
                if (!(lflags & KEY_LOOKUP_PARTIAL)) {
                        ret = wait_for_key_construction(key, true);
                        switch (ret) {
                        case -ERESTARTSYS:
                                goto invalid_key;
                        default:
                                if (need_perm != KEY_AUTHTOKEN_OVERRIDE &&
                                    need_perm != KEY_DEFER_PERM_CHECK)
                                        goto invalid_key;
                                break;
                        case 0:
                                break;
                        }
                } else if (need_perm != KEY_DEFER_PERM_CHECK) {
                        ret = key_validate(key);
                        if (ret < 0)
                                goto invalid_key;
                }

                ret = -EIO;
                if (!(lflags & KEY_LOOKUP_PARTIAL) &&
                    key_read_state(key) == KEY_IS_UNINSTANTIATED)
                        goto invalid_key;
        }

        /* check the permissions */
        ret = key_task_permission(key_ref, ctx.cred, need_perm);
        if (ret < 0)
                goto invalid_key;

        key->last_used_at = ktime_get_real_seconds();

error:
        put_cred(ctx.cred);
        return key_ref;

invalid_key:
        key_ref_put(key_ref);
        key_ref = ERR_PTR(ret);
        goto error;

        /* if we attempted to install a keyring, then it may have caused new
         * creds to be installed */
reget_creds:
        put_cred(ctx.cred);
        goto try_again;
}
EXPORT_SYMBOL(lookup_user_key);

/*
 * Join the named keyring as the session keyring if possible else attempt to
 * create a new one of that name and join that.
 *
 * If the name is NULL, an empty anonymous keyring will be installed as the
 * session keyring.
 *
 * Named session keyrings are joined with a semaphore held to prevent the
 * keyrings from going away whilst the attempt is made to going them and also
 * to prevent a race in creating compatible session keyrings.
 */
long join_session_keyring(const char *name)
{
        const struct cred *old;
        struct cred *new;
        struct key *keyring;
        long ret, serial;

        new = prepare_creds();
        if (!new)
                return -ENOMEM;
        old = current_cred();

        /* if no name is provided, install an anonymous keyring */
        if (!name) {
                ret = install_session_keyring_to_cred(new, NULL);
                if (ret < 0)
                        goto error;

                serial = new->session_keyring->serial;
                ret = commit_creds(new);
                if (ret == 0)
                        ret = serial;
                goto okay;
        }

        /* allow the user to join or create a named keyring */
        mutex_lock(&key_session_mutex);

        /* look for an existing keyring of this name */
        keyring = find_keyring_by_name(name, false);
        if (PTR_ERR(keyring) == -ENOKEY) {
                /* not found - try and create a new one */
                keyring = keyring_alloc(
                        name, old->uid, old->gid, old,
                        KEY_POS_ALL | KEY_USR_VIEW | KEY_USR_READ | KEY_USR_LINK,
                        KEY_ALLOC_IN_QUOTA, NULL, NULL);
                if (IS_ERR(keyring)) {
                        ret = PTR_ERR(keyring);
                        goto error2;
                }
        } else if (IS_ERR(keyring)) {
                ret = PTR_ERR(keyring);
                goto error2;
        } else if (keyring == new->session_keyring) {
                ret = 0;
                goto error3;
        }

        /* we've got a keyring - now to install it */
        ret = install_session_keyring_to_cred(new, keyring);
        if (ret < 0)
                goto error3;

        commit_creds(new);
        mutex_unlock(&key_session_mutex);

        ret = keyring->serial;
        key_put(keyring);
okay:
        return ret;

error3:
        key_put(keyring);
error2:
        mutex_unlock(&key_session_mutex);
error:
        abort_creds(new);
        return ret;
}

/*
 * Replace a process's session keyring on behalf of one of its children when
 * the target  process is about to resume userspace execution.
 */
void key_change_session_keyring(struct callback_head *twork)
{
        const struct cred *old = current_cred();
        struct cred *new = container_of(twork, struct cred, rcu);

        if (unlikely(current->flags & PF_EXITING)) {
                put_cred(new);
                return;
        }

        /* If get_ucounts fails more bits are needed in the refcount */
        if (unlikely(!get_ucounts(old->ucounts))) {
                WARN_ONCE(1, "In %s get_ucounts failed\n", __func__);
                put_cred(new);
                return;
        }

        new->  uid        = old->  uid;
        new-> euid        = old-> euid;
        new-> suid        = old-> suid;
        new->fsuid        = old->fsuid;
        new->  gid        = old->  gid;
        new-> egid        = old-> egid;
        new-> sgid        = old-> sgid;
        new->fsgid        = old->fsgid;
        new->user        = get_uid(old->user);
        new->ucounts        = old->ucounts;
        new->user_ns        = get_user_ns(old->user_ns);
        new->group_info        = get_group_info(old->group_info);

        new->securebits        = old->securebits;
        new->cap_inheritable        = old->cap_inheritable;
        new->cap_permitted        = old->cap_permitted;
        new->cap_effective        = old->cap_effective;
        new->cap_ambient        = old->cap_ambient;
        new->cap_bset                = old->cap_bset;

        new->jit_keyring        = old->jit_keyring;
        new->thread_keyring        = key_get(old->thread_keyring);
        new->process_keyring        = key_get(old->process_keyring);

        security_transfer_creds(new, old);

        commit_creds(new);
}

/*
 * Make sure that root's user and user-session keyrings exist.
 */
static int __init init_root_keyring(void)
{
        return look_up_user_keyrings(NULL, NULL);
}

late_initcall(init_root_keyring);


































































































































































































































































































































































































































































































































































   27 
   27 






















   27 




   27 











   27 






   27 


   27 















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Contiguous Memory Allocator
 *
 * Copyright (c) 2010-2011 by Samsung Electronics.
 * Copyright IBM Corporation, 2013
 * Copyright LG Electronics Inc., 2014
 * Written by:
 *        Marek Szyprowski <m.szyprowski@samsung.com>
 *        Michal Nazarewicz <mina86@mina86.com>
 *        Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
 *        Joonsoo Kim <iamjoonsoo.kim@lge.com>
 */

#define pr_fmt(fmt) "cma: " fmt

#define CREATE_TRACE_POINTS

#include <linux/memblock.h>
#include <linux/err.h>
#include <linux/mm.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <linux/log2.h>
#include <linux/cma.h>
#include <linux/highmem.h>
#include <linux/io.h>
#include <linux/kmemleak.h>
#include <trace/events/cma.h>

#include "internal.h"
#include "cma.h"

struct cma cma_areas[MAX_CMA_AREAS];
unsigned int cma_area_count;
static DEFINE_MUTEX(cma_mutex);

phys_addr_t cma_get_base(const struct cma *cma)
{
        return PFN_PHYS(cma->base_pfn);
}

unsigned long cma_get_size(const struct cma *cma)
{
        return cma->count << PAGE_SHIFT;
}

const char *cma_get_name(const struct cma *cma)
{
        return cma->name;
}

static unsigned long cma_bitmap_aligned_mask(const struct cma *cma,
                                             unsigned int align_order)
{
        if (align_order <= cma->order_per_bit)
                return 0;
        return (1UL << (align_order - cma->order_per_bit)) - 1;
}

/*
 * Find the offset of the base PFN from the specified align_order.
 * The value returned is represented in order_per_bits.
 */
static unsigned long cma_bitmap_aligned_offset(const struct cma *cma,
                                               unsigned int align_order)
{
        return (cma->base_pfn & ((1UL << align_order) - 1))
                >> cma->order_per_bit;
}

static unsigned long cma_bitmap_pages_to_bits(const struct cma *cma,
                                              unsigned long pages)
{
        return ALIGN(pages, 1UL << cma->order_per_bit) >> cma->order_per_bit;
}

static void cma_clear_bitmap(struct cma *cma, unsigned long pfn,
                             unsigned long count)
{
        unsigned long bitmap_no, bitmap_count;
        unsigned long flags;

        bitmap_no = (pfn - cma->base_pfn) >> cma->order_per_bit;
        bitmap_count = cma_bitmap_pages_to_bits(cma, count);

        spin_lock_irqsave(&cma->lock, flags);
        bitmap_clear(cma->bitmap, bitmap_no, bitmap_count);
        spin_unlock_irqrestore(&cma->lock, flags);
}

static void __init cma_activate_area(struct cma *cma)
{
        unsigned long base_pfn = cma->base_pfn, pfn;
        struct zone *zone;

        cma->bitmap = bitmap_zalloc(cma_bitmap_maxno(cma), GFP_KERNEL);
        if (!cma->bitmap)
                goto out_error;

        /*
         * alloc_contig_range() requires the pfn range specified to be in the
         * same zone. Simplify by forcing the entire CMA resv range to be in the
         * same zone.
         */
        WARN_ON_ONCE(!pfn_valid(base_pfn));
        zone = page_zone(pfn_to_page(base_pfn));
        for (pfn = base_pfn + 1; pfn < base_pfn + cma->count; pfn++) {
                WARN_ON_ONCE(!pfn_valid(pfn));
                if (page_zone(pfn_to_page(pfn)) != zone)
                        goto not_in_zone;
        }

        for (pfn = base_pfn; pfn < base_pfn + cma->count;
             pfn += pageblock_nr_pages)
                init_cma_reserved_pageblock(pfn_to_page(pfn));

        spin_lock_init(&cma->lock);

#ifdef CONFIG_CMA_DEBUGFS
        INIT_HLIST_HEAD(&cma->mem_head);
        spin_lock_init(&cma->mem_head_lock);
#endif

        return;

not_in_zone:
        bitmap_free(cma->bitmap);
out_error:
        /* Expose all pages to the buddy, they are useless for CMA. */
        if (!cma->reserve_pages_on_error) {
                for (pfn = base_pfn; pfn < base_pfn + cma->count; pfn++)
                        free_reserved_page(pfn_to_page(pfn));
        }
        totalcma_pages -= cma->count;
        cma->count = 0;
        pr_err("CMA area %s could not be activated\n", cma->name);
}

static int __init cma_init_reserved_areas(void)
{
        int i;

        for (i = 0; i < cma_area_count; i++)
                cma_activate_area(&cma_areas[i]);

        return 0;
}
core_initcall(cma_init_reserved_areas);

void __init cma_reserve_pages_on_error(struct cma *cma)
{
        cma->reserve_pages_on_error = true;
}

/**
 * cma_init_reserved_mem() - create custom contiguous area from reserved memory
 * @base: Base address of the reserved area
 * @size: Size of the reserved area (in bytes),
 * @order_per_bit: Order of pages represented by one bit on bitmap.
 * @name: The name of the area. If this parameter is NULL, the name of
 *        the area will be set to "cmaN", where N is a running counter of
 *        used areas.
 * @res_cma: Pointer to store the created cma region.
 *
 * This function creates custom contiguous area from already reserved memory.
 */
int __init cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
                                 unsigned int order_per_bit,
                                 const char *name,
                                 struct cma **res_cma)
{
        struct cma *cma;

        /* Sanity checks */
        if (cma_area_count == ARRAY_SIZE(cma_areas)) {
                pr_err("Not enough slots for CMA reserved regions!\n");
                return -ENOSPC;
        }

        if (!size || !memblock_is_region_reserved(base, size))
                return -EINVAL;

        /*
         * CMA uses CMA_MIN_ALIGNMENT_BYTES as alignment requirement which
         * needs pageblock_order to be initialized. Let's enforce it.
         */
        if (!pageblock_order) {
                pr_err("pageblock_order not yet initialized. Called during early boot?\n");
                return -EINVAL;
        }

        /* ensure minimal alignment required by mm core */
        if (!IS_ALIGNED(base | size, CMA_MIN_ALIGNMENT_BYTES))
                return -EINVAL;

        /*
         * Each reserved area must be initialised later, when more kernel
         * subsystems (like slab allocator) are available.
         */
        cma = &cma_areas[cma_area_count];

        if (name)
                snprintf(cma->name, CMA_MAX_NAME, name);
        else
                snprintf(cma->name, CMA_MAX_NAME,  "cma%d\n", cma_area_count);

        cma->base_pfn = PFN_DOWN(base);
        cma->count = size >> PAGE_SHIFT;
        cma->order_per_bit = order_per_bit;
        *res_cma = cma;
        cma_area_count++;
        totalcma_pages += cma->count;

        return 0;
}

/**
 * cma_declare_contiguous_nid() - reserve custom contiguous area
 * @base: Base address of the reserved area optional, use 0 for any
 * @size: Size of the reserved area (in bytes),
 * @limit: End address of the reserved memory (optional, 0 for any).
 * @alignment: Alignment for the CMA area, should be power of 2 or zero
 * @order_per_bit: Order of pages represented by one bit on bitmap.
 * @fixed: hint about where to place the reserved area
 * @name: The name of the area. See function cma_init_reserved_mem()
 * @res_cma: Pointer to store the created cma region.
 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
 *
 * This function reserves memory from early allocator. It should be
 * called by arch specific code once the early allocator (memblock or bootmem)
 * has been activated and all other subsystems have already allocated/reserved
 * memory. This function allows to create custom reserved areas.
 *
 * If @fixed is true, reserve contiguous area at exactly @base.  If false,
 * reserve in range from @base to @limit.
 */
int __init cma_declare_contiguous_nid(phys_addr_t base,
                        phys_addr_t size, phys_addr_t limit,
                        phys_addr_t alignment, unsigned int order_per_bit,
                        bool fixed, const char *name, struct cma **res_cma,
                        int nid)
{
        phys_addr_t memblock_end = memblock_end_of_DRAM();
        phys_addr_t highmem_start;
        int ret;

        /*
         * We can't use __pa(high_memory) directly, since high_memory
         * isn't a valid direct map VA, and DEBUG_VIRTUAL will (validly)
         * complain. Find the boundary by adding one to the last valid
         * address.
         */
        highmem_start = __pa(high_memory - 1) + 1;
        pr_debug("%s(size %pa, base %pa, limit %pa alignment %pa)\n",
                __func__, &size, &base, &limit, &alignment);

        if (cma_area_count == ARRAY_SIZE(cma_areas)) {
                pr_err("Not enough slots for CMA reserved regions!\n");
                return -ENOSPC;
        }

        if (!size)
                return -EINVAL;

        if (alignment && !is_power_of_2(alignment))
                return -EINVAL;

        if (!IS_ENABLED(CONFIG_NUMA))
                nid = NUMA_NO_NODE;

        /* Sanitise input arguments. */
        alignment = max_t(phys_addr_t, alignment, CMA_MIN_ALIGNMENT_BYTES);
        if (fixed && base & (alignment - 1)) {
                ret = -EINVAL;
                pr_err("Region at %pa must be aligned to %pa bytes\n",
                        &base, &alignment);
                goto err;
        }
        base = ALIGN(base, alignment);
        size = ALIGN(size, alignment);
        limit &= ~(alignment - 1);

        if (!base)
                fixed = false;

        /* size should be aligned with order_per_bit */
        if (!IS_ALIGNED(size >> PAGE_SHIFT, 1 << order_per_bit))
                return -EINVAL;

        /*
         * If allocating at a fixed base the request region must not cross the
         * low/high memory boundary.
         */
        if (fixed && base < highmem_start && base + size > highmem_start) {
                ret = -EINVAL;
                pr_err("Region at %pa defined on low/high memory boundary (%pa)\n",
                        &base, &highmem_start);
                goto err;
        }

        /*
         * If the limit is unspecified or above the memblock end, its effective
         * value will be the memblock end. Set it explicitly to simplify further
         * checks.
         */
        if (limit == 0 || limit > memblock_end)
                limit = memblock_end;

        if (base + size > limit) {
                ret = -EINVAL;
                pr_err("Size (%pa) of region at %pa exceeds limit (%pa)\n",
                        &size, &base, &limit);
                goto err;
        }

        /* Reserve memory */
        if (fixed) {
                if (memblock_is_region_reserved(base, size) ||
                    memblock_reserve(base, size) < 0) {
                        ret = -EBUSY;
                        goto err;
                }
        } else {
                phys_addr_t addr = 0;

                /*
                 * If there is enough memory, try a bottom-up allocation first.
                 * It will place the new cma area close to the start of the node
                 * and guarantee that the compaction is moving pages out of the
                 * cma area and not into it.
                 * Avoid using first 4GB to not interfere with constrained zones
                 * like DMA/DMA32.
                 */
#ifdef CONFIG_PHYS_ADDR_T_64BIT
                if (!memblock_bottom_up() && memblock_end >= SZ_4G + size) {
                        memblock_set_bottom_up(true);
                        addr = memblock_alloc_range_nid(size, alignment, SZ_4G,
                                                        limit, nid, true);
                        memblock_set_bottom_up(false);
                }
#endif

                /*
                 * All pages in the reserved area must come from the same zone.
                 * If the requested region crosses the low/high memory boundary,
                 * try allocating from high memory first and fall back to low
                 * memory in case of failure.
                 */
                if (!addr && base < highmem_start && limit > highmem_start) {
                        addr = memblock_alloc_range_nid(size, alignment,
                                        highmem_start, limit, nid, true);
                        limit = highmem_start;
                }

                if (!addr) {
                        addr = memblock_alloc_range_nid(size, alignment, base,
                                        limit, nid, true);
                        if (!addr) {
                                ret = -ENOMEM;
                                goto err;
                        }
                }

                /*
                 * kmemleak scans/reads tracked objects for pointers to other
                 * objects but this address isn't mapped and accessible
                 */
                kmemleak_ignore_phys(addr);
                base = addr;
        }

        ret = cma_init_reserved_mem(base, size, order_per_bit, name, res_cma);
        if (ret)
                goto free_mem;

        pr_info("Reserved %ld MiB at %pa on node %d\n", (unsigned long)size / SZ_1M,
                &base, nid);
        return 0;

free_mem:
        memblock_phys_free(base, size);
err:
        pr_err("Failed to reserve %ld MiB on node %d\n", (unsigned long)size / SZ_1M,
               nid);
        return ret;
}

static void cma_debug_show_areas(struct cma *cma)
{
        unsigned long next_zero_bit, next_set_bit, nr_zero;
        unsigned long start = 0;
        unsigned long nr_part, nr_total = 0;
        unsigned long nbits = cma_bitmap_maxno(cma);

        spin_lock_irq(&cma->lock);
        pr_info("number of available pages: ");
        for (;;) {
                next_zero_bit = find_next_zero_bit(cma->bitmap, nbits, start);
                if (next_zero_bit >= nbits)
                        break;
                next_set_bit = find_next_bit(cma->bitmap, nbits, next_zero_bit);
                nr_zero = next_set_bit - next_zero_bit;
                nr_part = nr_zero << cma->order_per_bit;
                pr_cont("%s%lu@%lu", nr_total ? "+" : "", nr_part,
                        next_zero_bit);
                nr_total += nr_part;
                start = next_zero_bit + nr_zero;
        }
        pr_cont("=> %lu free of %lu total pages\n", nr_total, cma->count);
        spin_unlock_irq(&cma->lock);
}

static struct page *__cma_alloc(struct cma *cma, unsigned long count,
                                unsigned int align, gfp_t gfp)
{
        unsigned long mask, offset;
        unsigned long pfn = -1;
        unsigned long start = 0;
        unsigned long bitmap_maxno, bitmap_no, bitmap_count;
        unsigned long i;
        struct page *page = NULL;
        int ret = -ENOMEM;
        const char *name = cma ? cma->name : NULL;

        trace_cma_alloc_start(name, count, align);

        if (!cma || !cma->count || !cma->bitmap)
                return page;

        pr_debug("%s(cma %p, name: %s, count %lu, align %d)\n", __func__,
                (void *)cma, cma->name, count, align);

        if (!count)
                return page;

        mask = cma_bitmap_aligned_mask(cma, align);
        offset = cma_bitmap_aligned_offset(cma, align);
        bitmap_maxno = cma_bitmap_maxno(cma);
        bitmap_count = cma_bitmap_pages_to_bits(cma, count);

        if (bitmap_count > bitmap_maxno)
                return page;

        for (;;) {
                spin_lock_irq(&cma->lock);
                bitmap_no = bitmap_find_next_zero_area_off(cma->bitmap,
                                bitmap_maxno, start, bitmap_count, mask,
                                offset);
                if (bitmap_no >= bitmap_maxno) {
                        spin_unlock_irq(&cma->lock);
                        break;
                }
                bitmap_set(cma->bitmap, bitmap_no, bitmap_count);
                /*
                 * It's safe to drop the lock here. We've marked this region for
                 * our exclusive use. If the migration fails we will take the
                 * lock again and unmark it.
                 */
                spin_unlock_irq(&cma->lock);

                pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);
                mutex_lock(&cma_mutex);
                ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp);
                mutex_unlock(&cma_mutex);
                if (ret == 0) {
                        page = pfn_to_page(pfn);
                        break;
                }

                cma_clear_bitmap(cma, pfn, count);
                if (ret != -EBUSY)
                        break;

                pr_debug("%s(): memory range at pfn 0x%lx %p is busy, retrying\n",
                         __func__, pfn, pfn_to_page(pfn));

                trace_cma_alloc_busy_retry(cma->name, pfn, pfn_to_page(pfn),
                                           count, align);
                /* try again with a bit different memory target */
                start = bitmap_no + mask + 1;
        }

        /*
         * CMA can allocate multiple page blocks, which results in different
         * blocks being marked with different tags. Reset the tags to ignore
         * those page blocks.
         */
        if (page) {
                for (i = 0; i < count; i++)
                        page_kasan_tag_reset(nth_page(page, i));
        }

        if (ret && !(gfp & __GFP_NOWARN)) {
                pr_err_ratelimited("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n",
                                   __func__, cma->name, count, ret);
                cma_debug_show_areas(cma);
        }

        pr_debug("%s(): returned %p\n", __func__, page);
        trace_cma_alloc_finish(name, pfn, page, count, align, ret);
        if (page) {
                count_vm_event(CMA_ALLOC_SUCCESS);
                cma_sysfs_account_success_pages(cma, count);
        } else {
                count_vm_event(CMA_ALLOC_FAIL);
                cma_sysfs_account_fail_pages(cma, count);
        }

        return page;
}

/**
 * cma_alloc() - allocate pages from contiguous area
 * @cma:   Contiguous memory region for which the allocation is performed.
 * @count: Requested number of pages.
 * @align: Requested alignment of pages (in PAGE_SIZE order).
 * @no_warn: Avoid printing message about failed allocation
 *
 * This function allocates part of contiguous memory on specific
 * contiguous memory area.
 */
struct page *cma_alloc(struct cma *cma, unsigned long count,
                       unsigned int align, bool no_warn)
{
        return __cma_alloc(cma, count, align, GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
}

struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp)
{
        struct page *page;

        if (WARN_ON(!order || !(gfp & __GFP_COMP)))
                return NULL;

        page = __cma_alloc(cma, 1 << order, order, gfp);

        return page ? page_folio(page) : NULL;
}

bool cma_pages_valid(struct cma *cma, const struct page *pages,
                     unsigned long count)
{
        unsigned long pfn;

        if (!cma || !pages)
                return false;

        pfn = page_to_pfn(pages);

        if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count) {
                pr_debug("%s(page %p, count %lu)\n", __func__,
                                                (void *)pages, count);
                return false;
        }

        return true;
}

/**
 * cma_release() - release allocated pages
 * @cma:   Contiguous memory region for which the allocation is performed.
 * @pages: Allocated pages.
 * @count: Number of allocated pages.
 *
 * This function releases memory allocated by cma_alloc().
 * It returns false when provided pages do not belong to contiguous area and
 * true otherwise.
 */
bool cma_release(struct cma *cma, const struct page *pages,
                 unsigned long count)
{
        unsigned long pfn;

        if (!cma_pages_valid(cma, pages, count))
                return false;

        pr_debug("%s(page %p, count %lu)\n", __func__, (void *)pages, count);

        pfn = page_to_pfn(pages);

        VM_BUG_ON(pfn + count > cma->base_pfn + cma->count);

        free_contig_range(pfn, count);
        cma_clear_bitmap(cma, pfn, count);
        cma_sysfs_account_release_pages(cma, count);
        trace_cma_release(cma->name, pfn, pages, count);

        return true;
}

bool cma_free_folio(struct cma *cma, const struct folio *folio)
{
        if (WARN_ON(!folio_test_large(folio)))
                return false;

        return cma_release(cma, &folio->page, folio_nr_pages(folio));
}

int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data)
{
        int i;

        for (i = 0; i < cma_area_count; i++) {
                int ret = it(&cma_areas[i], data);

                if (ret)
                        return ret;
        }

        return 0;
}






























































































































































































































































































    9 


   14 


   14 












    9 


    9 


    9 








    5 


   13 



















    1 






    1 


    1 





















   68 

















   68 

























































































































   68 








   73 
   73 
   73 
   73 
   73 
   73 

   73 
   73 



   73 

   73 

   73 

   73 


   73 























































   68 










































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Linux NET3:        IP/IP protocol decoder.
 *
 *        Authors:
 *                Sam Lantinga (slouken@cs.ucdavis.edu)  02/01/95
 *
 *        Fixes:
 *                Alan Cox        :        Merged and made usable non modular (its so tiny its silly as
 *                                        a module taking up 2 pages).
 *                Alan Cox        :         Fixed bug with 1.3.18 and IPIP not working (now needs to set skb->h.iph)
 *                                        to keep ip_forward happy.
 *                Alan Cox        :        More fixes for 1.3.21, and firewall fix. Maybe this will work soon 8).
 *                Kai Schulte        :        Fixed #defines for IP_FIREWALL->FIREWALL
 *              David Woodhouse :       Perform some basic ICMP handling.
 *                                      IPIP Routing without decapsulation.
 *              Carlos Picoto   :       GRE over IP support
 *                Alexey Kuznetsov:        Reworked. Really, now it is truncated version of ipv4/ip_gre.c.
 *                                        I do not want to merge them together.
 */

/* tunnel.c: an IP tunnel driver

        The purpose of this driver is to provide an IP tunnel through
        which you can tunnel network traffic transparently across subnets.

        This was written by looking at Nick Holloway's dummy driver
        Thanks for the great code!

                -Sam Lantinga        (slouken@cs.ucdavis.edu)  02/01/95

        Minor tweaks:
                Cleaned up the code a little and added some pre-1.3.0 tweaks.
                dev->hard_header/hard_header_len changed to use no headers.
                Comments/bracketing tweaked.
                Made the tunnels use dev->name not tunnel: when error reporting.
                Added tx_dropped stat

                -Alan Cox        (alan@lxorguk.ukuu.org.uk) 21 March 95

        Reworked:
                Changed to tunnel to destination gateway in addition to the
                        tunnel's pointopoint address
                Almost completely rewritten
                Note:  There is currently no firewall or ICMP handling done.

                -Sam Lantinga        (slouken@cs.ucdavis.edu) 02/13/96

*/

/* Things I wish I had known when writing the tunnel driver:

        When the tunnel_xmit() function is called, the skb contains the
        packet to be sent (plus a great deal of extra info), and dev
        contains the tunnel device that _we_ are.

        When we are passed a packet, we are expected to fill in the
        source address with our source IP address.

        What is the proper way to allocate, copy and free a buffer?
        After you allocate it, it is a "0 length" chunk of memory
        starting at zero.  If you want to add headers to the buffer
        later, you'll have to call "skb_reserve(skb, amount)" with
        the amount of memory you want reserved.  Then, you call
        "skb_put(skb, amount)" with the amount of space you want in
        the buffer.  skb_put() returns a pointer to the top (#0) of
        that buffer.  skb->len is set to the amount of space you have
        "allocated" with skb_put().  You can then write up to skb->len
        bytes to that buffer.  If you need more, you can call skb_put()
        again with the additional amount of space you need.  You can
        find out how much more space you can allocate by calling
        "skb_tailroom(skb)".
        Now, to add header space, call "skb_push(skb, header_len)".
        This creates space at the beginning of the buffer and returns
        a pointer to this new space.  If later you need to strip a
        header from a buffer, call "skb_pull(skb, header_len)".
        skb_headroom() will return how much space is left at the top
        of the buffer (before the main data).  Remember, this headroom
        space must be reserved before the skb_put() function is called.
        */

/*
   This version of net/ipv4/ipip.c is cloned of net/ipv4/ip_gre.c

   For comments look at net/ipv4/ip_gre.c --ANK
 */


#include <linux/capability.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/in.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/if_arp.h>
#include <linux/init.h>
#include <linux/netfilter_ipv4.h>
#include <linux/if_ether.h>

#include <net/sock.h>
#include <net/ip.h>
#include <net/icmp.h>
#include <net/ip_tunnels.h>
#include <net/inet_ecn.h>
#include <net/xfrm.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/dst_metadata.h>

static bool log_ecn_error = true;
module_param(log_ecn_error, bool, 0644);
MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");

static unsigned int ipip_net_id __read_mostly;

static int ipip_tunnel_init(struct net_device *dev);
static struct rtnl_link_ops ipip_link_ops __read_mostly;

static int ipip_err(struct sk_buff *skb, u32 info)
{
        /* All the routers (except for Linux) return only
         * 8 bytes of packet payload. It means, that precise relaying of
         * ICMP in the real Internet is absolutely infeasible.
         */
        struct net *net = dev_net(skb->dev);
        struct ip_tunnel_net *itn = net_generic(net, ipip_net_id);
        const struct iphdr *iph = (const struct iphdr *)skb->data;
        IP_TUNNEL_DECLARE_FLAGS(flags) = { };
        const int type = icmp_hdr(skb)->type;
        const int code = icmp_hdr(skb)->code;
        struct ip_tunnel *t;
        int err = 0;

        __set_bit(IP_TUNNEL_NO_KEY_BIT, flags);

        t = ip_tunnel_lookup(itn, skb->dev->ifindex, flags, iph->daddr,
                             iph->saddr, 0);
        if (!t) {
                err = -ENOENT;
                goto out;
        }

        switch (type) {
        case ICMP_DEST_UNREACH:
                switch (code) {
                case ICMP_SR_FAILED:
                        /* Impossible event. */
                        goto out;
                default:
                        /* All others are translated to HOST_UNREACH.
                         * rfc2003 contains "deep thoughts" about NET_UNREACH,
                         * I believe they are just ether pollution. --ANK
                         */
                        break;
                }
                break;

        case ICMP_TIME_EXCEEDED:
                if (code != ICMP_EXC_TTL)
                        goto out;
                break;

        case ICMP_REDIRECT:
                break;

        default:
                goto out;
        }

        if (type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED) {
                ipv4_update_pmtu(skb, net, info, t->parms.link, iph->protocol);
                goto out;
        }

        if (type == ICMP_REDIRECT) {
                ipv4_redirect(skb, net, t->parms.link, iph->protocol);
                goto out;
        }

        if (t->parms.iph.daddr == 0) {
                err = -ENOENT;
                goto out;
        }

        if (t->parms.iph.ttl == 0 && type == ICMP_TIME_EXCEEDED)
                goto out;

        if (time_before(jiffies, t->err_time + IPTUNNEL_ERR_TIMEO))
                t->err_count++;
        else
                t->err_count = 1;
        t->err_time = jiffies;

out:
        return err;
}

static const struct tnl_ptk_info ipip_tpi = {
        /* no tunnel info required for ipip. */
        .proto = htons(ETH_P_IP),
};

#if IS_ENABLED(CONFIG_MPLS)
static const struct tnl_ptk_info mplsip_tpi = {
        /* no tunnel info required for mplsip. */
        .proto = htons(ETH_P_MPLS_UC),
};
#endif

static int ipip_tunnel_rcv(struct sk_buff *skb, u8 ipproto)
{
        struct net *net = dev_net(skb->dev);
        struct ip_tunnel_net *itn = net_generic(net, ipip_net_id);
        IP_TUNNEL_DECLARE_FLAGS(flags) = { };
        struct metadata_dst *tun_dst = NULL;
        struct ip_tunnel *tunnel;
        const struct iphdr *iph;

        __set_bit(IP_TUNNEL_NO_KEY_BIT, flags);

        iph = ip_hdr(skb);
        tunnel = ip_tunnel_lookup(itn, skb->dev->ifindex, flags, iph->saddr,
                                  iph->daddr, 0);
        if (tunnel) {
                const struct tnl_ptk_info *tpi;

                if (tunnel->parms.iph.protocol != ipproto &&
                    tunnel->parms.iph.protocol != 0)
                        goto drop;

                if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
                        goto drop;
#if IS_ENABLED(CONFIG_MPLS)
                if (ipproto == IPPROTO_MPLS)
                        tpi = &mplsip_tpi;
                else
#endif
                        tpi = &ipip_tpi;
                if (iptunnel_pull_header(skb, 0, tpi->proto, false))
                        goto drop;
                if (tunnel->collect_md) {
                        ip_tunnel_flags_zero(flags);

                        tun_dst = ip_tun_rx_dst(skb, flags, 0, 0);
                        if (!tun_dst)
                                return 0;
                        ip_tunnel_md_udp_encap(skb, &tun_dst->u.tun_info);
                }
                skb_reset_mac_header(skb);

                return ip_tunnel_rcv(tunnel, skb, tpi, tun_dst, log_ecn_error);
        }

        return -1;

drop:
        kfree_skb(skb);
        return 0;
}

static int ipip_rcv(struct sk_buff *skb)
{
        return ipip_tunnel_rcv(skb, IPPROTO_IPIP);
}

#if IS_ENABLED(CONFIG_MPLS)
static int mplsip_rcv(struct sk_buff *skb)
{
        return ipip_tunnel_rcv(skb, IPPROTO_MPLS);
}
#endif

/*
 *        This function assumes it is being called from dev_queue_xmit()
 *        and that skb is filled properly by that function.
 */
static netdev_tx_t ipip_tunnel_xmit(struct sk_buff *skb,
                                    struct net_device *dev)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        const struct iphdr  *tiph = &tunnel->parms.iph;
        u8 ipproto;

        if (!pskb_inet_may_pull(skb))
                goto tx_error;

        switch (skb->protocol) {
        case htons(ETH_P_IP):
                ipproto = IPPROTO_IPIP;
                break;
#if IS_ENABLED(CONFIG_MPLS)
        case htons(ETH_P_MPLS_UC):
                ipproto = IPPROTO_MPLS;
                break;
#endif
        default:
                goto tx_error;
        }

        if (tiph->protocol != ipproto && tiph->protocol != 0)
                goto tx_error;

        if (iptunnel_handle_offloads(skb, SKB_GSO_IPXIP4))
                goto tx_error;

        skb_set_inner_ipproto(skb, ipproto);

        if (tunnel->collect_md)
                ip_md_tunnel_xmit(skb, dev, ipproto, 0);
        else
                ip_tunnel_xmit(skb, dev, tiph, ipproto);
        return NETDEV_TX_OK;

tx_error:
        kfree_skb(skb);

        DEV_STATS_INC(dev, tx_errors);
        return NETDEV_TX_OK;
}

static bool ipip_tunnel_ioctl_verify_protocol(u8 ipproto)
{
        switch (ipproto) {
        case 0:
        case IPPROTO_IPIP:
#if IS_ENABLED(CONFIG_MPLS)
        case IPPROTO_MPLS:
#endif
                return true;
        }

        return false;
}

static int
ipip_tunnel_ctl(struct net_device *dev, struct ip_tunnel_parm_kern *p, int cmd)
{
        if (cmd == SIOCADDTUNNEL || cmd == SIOCCHGTUNNEL) {
                if (p->iph.version != 4 ||
                    !ipip_tunnel_ioctl_verify_protocol(p->iph.protocol) ||
                    p->iph.ihl != 5 || (p->iph.frag_off & htons(~IP_DF)))
                        return -EINVAL;
        }

        p->i_key = p->o_key = 0;
        ip_tunnel_flags_zero(p->i_flags);
        ip_tunnel_flags_zero(p->o_flags);
        return ip_tunnel_ctl(dev, p, cmd);
}

static const struct net_device_ops ipip_netdev_ops = {
        .ndo_init       = ipip_tunnel_init,
        .ndo_uninit     = ip_tunnel_uninit,
        .ndo_start_xmit        = ipip_tunnel_xmit,
        .ndo_siocdevprivate = ip_tunnel_siocdevprivate,
        .ndo_change_mtu = ip_tunnel_change_mtu,
        .ndo_get_stats64 = dev_get_tstats64,
        .ndo_get_iflink = ip_tunnel_get_iflink,
        .ndo_tunnel_ctl        = ipip_tunnel_ctl,
};

#define IPIP_FEATURES (NETIF_F_SG |                \
                       NETIF_F_FRAGLIST |        \
                       NETIF_F_HIGHDMA |        \
                       NETIF_F_GSO_SOFTWARE |        \
                       NETIF_F_HW_CSUM)

static void ipip_tunnel_setup(struct net_device *dev)
{
        dev->netdev_ops                = &ipip_netdev_ops;
        dev->header_ops                = &ip_tunnel_header_ops;

        dev->type                = ARPHRD_TUNNEL;
        dev->flags                = IFF_NOARP;
        dev->addr_len                = 4;
        dev->lltx                = true;
        netif_keep_dst(dev);

        dev->features                |= IPIP_FEATURES;
        dev->hw_features        |= IPIP_FEATURES;
        ip_tunnel_setup(dev, ipip_net_id);
}

static int ipip_tunnel_init(struct net_device *dev)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);

        __dev_addr_set(dev, &tunnel->parms.iph.saddr, 4);
        memcpy(dev->broadcast, &tunnel->parms.iph.daddr, 4);

        tunnel->tun_hlen = 0;
        tunnel->hlen = tunnel->tun_hlen + tunnel->encap_hlen;
        return ip_tunnel_init(dev);
}

static int ipip_tunnel_validate(struct nlattr *tb[], struct nlattr *data[],
                                struct netlink_ext_ack *extack)
{
        u8 proto;

        if (!data || !data[IFLA_IPTUN_PROTO])
                return 0;

        proto = nla_get_u8(data[IFLA_IPTUN_PROTO]);
        if (proto != IPPROTO_IPIP && proto != IPPROTO_MPLS && proto != 0)
                return -EINVAL;

        return 0;
}

static void ipip_netlink_parms(struct nlattr *data[],
                               struct ip_tunnel_parm_kern *parms,
                               bool *collect_md, __u32 *fwmark)
{
        memset(parms, 0, sizeof(*parms));

        parms->iph.version = 4;
        parms->iph.protocol = IPPROTO_IPIP;
        parms->iph.ihl = 5;
        *collect_md = false;

        if (!data)
                return;

        ip_tunnel_netlink_parms(data, parms);

        if (data[IFLA_IPTUN_COLLECT_METADATA])
                *collect_md = true;

        if (data[IFLA_IPTUN_FWMARK])
                *fwmark = nla_get_u32(data[IFLA_IPTUN_FWMARK]);
}

static int ipip_newlink(struct net *src_net, struct net_device *dev,
                        struct nlattr *tb[], struct nlattr *data[],
                        struct netlink_ext_ack *extack)
{
        struct ip_tunnel *t = netdev_priv(dev);
        struct ip_tunnel_encap ipencap;
        struct ip_tunnel_parm_kern p;
        __u32 fwmark = 0;

        if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
                int err = ip_tunnel_encap_setup(t, &ipencap);

                if (err < 0)
                        return err;
        }

        ipip_netlink_parms(data, &p, &t->collect_md, &fwmark);
        return ip_tunnel_newlink(dev, tb, &p, fwmark);
}

static int ipip_changelink(struct net_device *dev, struct nlattr *tb[],
                           struct nlattr *data[],
                           struct netlink_ext_ack *extack)
{
        struct ip_tunnel *t = netdev_priv(dev);
        struct ip_tunnel_encap ipencap;
        struct ip_tunnel_parm_kern p;
        bool collect_md;
        __u32 fwmark = t->fwmark;

        if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
                int err = ip_tunnel_encap_setup(t, &ipencap);

                if (err < 0)
                        return err;
        }

        ipip_netlink_parms(data, &p, &collect_md, &fwmark);
        if (collect_md)
                return -EINVAL;

        if (((dev->flags & IFF_POINTOPOINT) && !p.iph.daddr) ||
            (!(dev->flags & IFF_POINTOPOINT) && p.iph.daddr))
                return -EINVAL;

        return ip_tunnel_changelink(dev, tb, &p, fwmark);
}

static size_t ipip_get_size(const struct net_device *dev)
{
        return
                /* IFLA_IPTUN_LINK */
                nla_total_size(4) +
                /* IFLA_IPTUN_LOCAL */
                nla_total_size(4) +
                /* IFLA_IPTUN_REMOTE */
                nla_total_size(4) +
                /* IFLA_IPTUN_TTL */
                nla_total_size(1) +
                /* IFLA_IPTUN_TOS */
                nla_total_size(1) +
                /* IFLA_IPTUN_PROTO */
                nla_total_size(1) +
                /* IFLA_IPTUN_PMTUDISC */
                nla_total_size(1) +
                /* IFLA_IPTUN_ENCAP_TYPE */
                nla_total_size(2) +
                /* IFLA_IPTUN_ENCAP_FLAGS */
                nla_total_size(2) +
                /* IFLA_IPTUN_ENCAP_SPORT */
                nla_total_size(2) +
                /* IFLA_IPTUN_ENCAP_DPORT */
                nla_total_size(2) +
                /* IFLA_IPTUN_COLLECT_METADATA */
                nla_total_size(0) +
                /* IFLA_IPTUN_FWMARK */
                nla_total_size(4) +
                0;
}

static int ipip_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        struct ip_tunnel_parm_kern *parm = &tunnel->parms;

        if (nla_put_u32(skb, IFLA_IPTUN_LINK, parm->link) ||
            nla_put_in_addr(skb, IFLA_IPTUN_LOCAL, parm->iph.saddr) ||
            nla_put_in_addr(skb, IFLA_IPTUN_REMOTE, parm->iph.daddr) ||
            nla_put_u8(skb, IFLA_IPTUN_TTL, parm->iph.ttl) ||
            nla_put_u8(skb, IFLA_IPTUN_TOS, parm->iph.tos) ||
            nla_put_u8(skb, IFLA_IPTUN_PROTO, parm->iph.protocol) ||
            nla_put_u8(skb, IFLA_IPTUN_PMTUDISC,
                       !!(parm->iph.frag_off & htons(IP_DF))) ||
            nla_put_u32(skb, IFLA_IPTUN_FWMARK, tunnel->fwmark))
                goto nla_put_failure;

        if (nla_put_u16(skb, IFLA_IPTUN_ENCAP_TYPE,
                        tunnel->encap.type) ||
            nla_put_be16(skb, IFLA_IPTUN_ENCAP_SPORT,
                         tunnel->encap.sport) ||
            nla_put_be16(skb, IFLA_IPTUN_ENCAP_DPORT,
                         tunnel->encap.dport) ||
            nla_put_u16(skb, IFLA_IPTUN_ENCAP_FLAGS,
                        tunnel->encap.flags))
                goto nla_put_failure;

        if (tunnel->collect_md)
                if (nla_put_flag(skb, IFLA_IPTUN_COLLECT_METADATA))
                        goto nla_put_failure;
        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static const struct nla_policy ipip_policy[IFLA_IPTUN_MAX + 1] = {
        [IFLA_IPTUN_LINK]                = { .type = NLA_U32 },
        [IFLA_IPTUN_LOCAL]                = { .type = NLA_U32 },
        [IFLA_IPTUN_REMOTE]                = { .type = NLA_U32 },
        [IFLA_IPTUN_TTL]                = { .type = NLA_U8 },
        [IFLA_IPTUN_TOS]                = { .type = NLA_U8 },
        [IFLA_IPTUN_PROTO]                = { .type = NLA_U8 },
        [IFLA_IPTUN_PMTUDISC]                = { .type = NLA_U8 },
        [IFLA_IPTUN_ENCAP_TYPE]                = { .type = NLA_U16 },
        [IFLA_IPTUN_ENCAP_FLAGS]        = { .type = NLA_U16 },
        [IFLA_IPTUN_ENCAP_SPORT]        = { .type = NLA_U16 },
        [IFLA_IPTUN_ENCAP_DPORT]        = { .type = NLA_U16 },
        [IFLA_IPTUN_COLLECT_METADATA]        = { .type = NLA_FLAG },
        [IFLA_IPTUN_FWMARK]                = { .type = NLA_U32 },
};

static struct rtnl_link_ops ipip_link_ops __read_mostly = {
        .kind                = "ipip",
        .maxtype        = IFLA_IPTUN_MAX,
        .policy                = ipip_policy,
        .priv_size        = sizeof(struct ip_tunnel),
        .setup                = ipip_tunnel_setup,
        .validate        = ipip_tunnel_validate,
        .newlink        = ipip_newlink,
        .changelink        = ipip_changelink,
        .dellink        = ip_tunnel_dellink,
        .get_size        = ipip_get_size,
        .fill_info        = ipip_fill_info,
        .get_link_net        = ip_tunnel_get_link_net,
};

static struct xfrm_tunnel ipip_handler __read_mostly = {
        .handler        =        ipip_rcv,
        .err_handler        =        ipip_err,
        .priority        =        1,
};

#if IS_ENABLED(CONFIG_MPLS)
static struct xfrm_tunnel mplsip_handler __read_mostly = {
        .handler        =        mplsip_rcv,
        .err_handler        =        ipip_err,
        .priority        =        1,
};
#endif

static int __net_init ipip_init_net(struct net *net)
{
        return ip_tunnel_init_net(net, ipip_net_id, &ipip_link_ops, "tunl0");
}

static void __net_exit ipip_exit_batch_rtnl(struct list_head *list_net,
                                            struct list_head *dev_to_kill)
{
        ip_tunnel_delete_nets(list_net, ipip_net_id, &ipip_link_ops,
                              dev_to_kill);
}

static struct pernet_operations ipip_net_ops = {
        .init = ipip_init_net,
        .exit_batch_rtnl = ipip_exit_batch_rtnl,
        .id   = &ipip_net_id,
        .size = sizeof(struct ip_tunnel_net),
};

static int __init ipip_init(void)
{
        int err;

        pr_info("ipip: IPv4 and MPLS over IPv4 tunneling driver\n");

        err = register_pernet_device(&ipip_net_ops);
        if (err < 0)
                return err;
        err = xfrm4_tunnel_register(&ipip_handler, AF_INET);
        if (err < 0) {
                pr_info("%s: can't register tunnel\n", __func__);
                goto xfrm_tunnel_ipip_failed;
        }
#if IS_ENABLED(CONFIG_MPLS)
        err = xfrm4_tunnel_register(&mplsip_handler, AF_MPLS);
        if (err < 0) {
                pr_info("%s: can't register tunnel\n", __func__);
                goto xfrm_tunnel_mplsip_failed;
        }
#endif
        err = rtnl_link_register(&ipip_link_ops);
        if (err < 0)
                goto rtnl_link_failed;

out:
        return err;

rtnl_link_failed:
#if IS_ENABLED(CONFIG_MPLS)
        xfrm4_tunnel_deregister(&mplsip_handler, AF_MPLS);
xfrm_tunnel_mplsip_failed:

#endif
        xfrm4_tunnel_deregister(&ipip_handler, AF_INET);
xfrm_tunnel_ipip_failed:
        unregister_pernet_device(&ipip_net_ops);
        goto out;
}

static void __exit ipip_fini(void)
{
        rtnl_link_unregister(&ipip_link_ops);
        if (xfrm4_tunnel_deregister(&ipip_handler, AF_INET))
                pr_info("%s: can't deregister tunnel\n", __func__);
#if IS_ENABLED(CONFIG_MPLS)
        if (xfrm4_tunnel_deregister(&mplsip_handler, AF_MPLS))
                pr_info("%s: can't deregister tunnel\n", __func__);
#endif
        unregister_pernet_device(&ipip_net_ops);
}

module_init(ipip_init);
module_exit(ipip_fini);
MODULE_DESCRIPTION("IP/IP protocol decoder library");
MODULE_LICENSE("GPL");
MODULE_ALIAS_RTNL_LINK("ipip");
MODULE_ALIAS_NETDEV("tunl0");































































































































































































































































    1 





    1 














    1 











    1 














    1 

    1 
















    1 







    1 

    1 



    1 

    1 
    1 



    1 



    1 












































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
 */

#include <net/genetlink.h>
#define CREATE_TRACE_POINTS
#include <trace/events/devlink.h>

#include "devl_internal.h"

EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);

DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);

static struct devlink *devlinks_xa_get(unsigned long index)
{
        struct devlink *devlink;

        rcu_read_lock();
        devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED);
        if (!devlink || !devlink_try_get(devlink))
                devlink = NULL;
        rcu_read_unlock();
        return devlink;
}

/* devlink_rels xarray contains 1:1 relationships between
 * devlink object and related nested devlink instance.
 * The xarray index is used to get the nested object from
 * the nested-in object code.
 */
static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1);

#define DEVLINK_REL_IN_USE XA_MARK_0

struct devlink_rel {
        u32 index;
        refcount_t refcount;
        u32 devlink_index;
        struct {
                u32 devlink_index;
                u32 obj_index;
                devlink_rel_notify_cb_t *notify_cb;
                devlink_rel_cleanup_cb_t *cleanup_cb;
                struct delayed_work notify_work;
        } nested_in;
};

static void devlink_rel_free(struct devlink_rel *rel)
{
        xa_erase(&devlink_rels, rel->index);
        kfree(rel);
}

static void __devlink_rel_get(struct devlink_rel *rel)
{
        refcount_inc(&rel->refcount);
}

static void __devlink_rel_put(struct devlink_rel *rel)
{
        if (refcount_dec_and_test(&rel->refcount))
                devlink_rel_free(rel);
}

static void devlink_rel_nested_in_notify_work(struct work_struct *work)
{
        struct devlink_rel *rel = container_of(work, struct devlink_rel,
                                               nested_in.notify_work.work);
        struct devlink *devlink;

        devlink = devlinks_xa_get(rel->nested_in.devlink_index);
        if (!devlink)
                goto rel_put;
        if (!devl_trylock(devlink)) {
                devlink_put(devlink);
                goto reschedule_work;
        }
        if (!devl_is_registered(devlink)) {
                devl_unlock(devlink);
                devlink_put(devlink);
                goto rel_put;
        }
        if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE))
                rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index);
        rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index);
        devl_unlock(devlink);
        devlink_put(devlink);

rel_put:
        __devlink_rel_put(rel);
        return;

reschedule_work:
        schedule_delayed_work(&rel->nested_in.notify_work, 1);
}

static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel)
{
        __devlink_rel_get(rel);
        schedule_delayed_work(&rel->nested_in.notify_work, 0);
}

static struct devlink_rel *devlink_rel_alloc(void)
{
        struct devlink_rel *rel;
        static u32 next;
        int err;

        rel = kzalloc(sizeof(*rel), GFP_KERNEL);
        if (!rel)
                return ERR_PTR(-ENOMEM);

        err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel,
                              xa_limit_32b, &next, GFP_KERNEL);
        if (err) {
                kfree(rel);
                return ERR_PTR(err);
        }

        refcount_set(&rel->refcount, 1);
        INIT_DELAYED_WORK(&rel->nested_in.notify_work,
                          &devlink_rel_nested_in_notify_work);
        return rel;
}

static void devlink_rel_put(struct devlink *devlink)
{
        struct devlink_rel *rel = devlink->rel;

        if (!rel)
                return;
        xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
        devlink_rel_nested_in_notify_work_schedule(rel);
        __devlink_rel_put(rel);
        devlink->rel = NULL;
}

void devlink_rel_nested_in_clear(u32 rel_index)
{
        xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE);
}

int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index,
                              u32 obj_index, devlink_rel_notify_cb_t *notify_cb,
                              devlink_rel_cleanup_cb_t *cleanup_cb,
                              struct devlink *devlink)
{
        struct devlink_rel *rel = devlink_rel_alloc();

        ASSERT_DEVLINK_NOT_REGISTERED(devlink);

        if (IS_ERR(rel))
                return PTR_ERR(rel);

        rel->devlink_index = devlink->index;
        rel->nested_in.devlink_index = devlink_index;
        rel->nested_in.obj_index = obj_index;
        rel->nested_in.notify_cb = notify_cb;
        rel->nested_in.cleanup_cb = cleanup_cb;
        *rel_index = rel->index;
        xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
        devlink->rel = rel;
        return 0;
}

/**
 * devlink_rel_nested_in_notify - Notify the object this devlink
 *                                  instance is nested in.
 * @devlink: devlink
 *
 * This is called upon network namespace change of devlink instance.
 * In case this devlink instance is nested in another devlink object,
 * a notification of a change of this object should be sent
 * over netlink. The parent devlink instance lock needs to be
 * taken during the notification preparation.
 * However, since the devlink lock of nested instance is held here,
 * we would end with wrong devlink instance lock ordering and
 * deadlock. Therefore the work is utilized to avoid that.
 */
void devlink_rel_nested_in_notify(struct devlink *devlink)
{
        struct devlink_rel *rel = devlink->rel;

        if (!rel)
                return;
        devlink_rel_nested_in_notify_work_schedule(rel);
}

static struct devlink_rel *devlink_rel_find(unsigned long rel_index)
{
        return xa_find(&devlink_rels, &rel_index, rel_index,
                       DEVLINK_REL_IN_USE);
}

static struct devlink *devlink_rel_devlink_get(u32 rel_index)
{
        struct devlink_rel *rel;
        u32 devlink_index;

        if (!rel_index)
                return NULL;
        xa_lock(&devlink_rels);
        rel = devlink_rel_find(rel_index);
        if (rel)
                devlink_index = rel->devlink_index;
        xa_unlock(&devlink_rels);
        if (!rel)
                return NULL;
        return devlinks_xa_get(devlink_index);
}

int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink,
                                   u32 rel_index, int attrtype,
                                   bool *msg_updated)
{
        struct net *net = devlink_net(devlink);
        struct devlink *rel_devlink;
        int err;

        rel_devlink = devlink_rel_devlink_get(rel_index);
        if (!rel_devlink)
                return 0;
        err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype);
        devlink_put(rel_devlink);
        if (!err && msg_updated)
                *msg_updated = true;
        return err;
}

void *devlink_priv(struct devlink *devlink)
{
        return &devlink->priv;
}
EXPORT_SYMBOL_GPL(devlink_priv);

struct devlink *priv_to_devlink(void *priv)
{
        return container_of(priv, struct devlink, priv);
}
EXPORT_SYMBOL_GPL(priv_to_devlink);

struct device *devlink_to_dev(const struct devlink *devlink)
{
        return devlink->dev;
}
EXPORT_SYMBOL_GPL(devlink_to_dev);

struct net *devlink_net(const struct devlink *devlink)
{
        return read_pnet(&devlink->_net);
}
EXPORT_SYMBOL_GPL(devlink_net);

void devl_assert_locked(struct devlink *devlink)
{
        lockdep_assert_held(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_assert_locked);

#ifdef CONFIG_LOCKDEP
/* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
bool devl_lock_is_held(struct devlink *devlink)
{
        return lockdep_is_held(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_lock_is_held);
#endif

void devl_lock(struct devlink *devlink)
{
        mutex_lock(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_lock);

int devl_trylock(struct devlink *devlink)
{
        return mutex_trylock(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_trylock);

void devl_unlock(struct devlink *devlink)
{
        mutex_unlock(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_unlock);

/**
 * devlink_try_get() - try to obtain a reference on a devlink instance
 * @devlink: instance to reference
 *
 * Obtain a reference on a devlink instance. A reference on a devlink instance
 * only implies that it's safe to take the instance lock. It does not imply
 * that the instance is registered, use devl_is_registered() after taking
 * the instance lock to check registration status.
 */
struct devlink *__must_check devlink_try_get(struct devlink *devlink)
{
        if (refcount_inc_not_zero(&devlink->refcount))
                return devlink;
        return NULL;
}

static void devlink_release(struct work_struct *work)
{
        struct devlink *devlink;

        devlink = container_of(to_rcu_work(work), struct devlink, rwork);

        mutex_destroy(&devlink->lock);
        lockdep_unregister_key(&devlink->lock_key);
        put_device(devlink->dev);
        kvfree(devlink);
}

void devlink_put(struct devlink *devlink)
{
        if (refcount_dec_and_test(&devlink->refcount))
                queue_rcu_work(system_wq, &devlink->rwork);
}

struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp)
{
        struct devlink *devlink = NULL;

        rcu_read_lock();
retry:
        devlink = xa_find(&devlinks, indexp, ULONG_MAX, DEVLINK_REGISTERED);
        if (!devlink)
                goto unlock;

        if (!devlink_try_get(devlink))
                goto next;
        if (!net_eq(devlink_net(devlink), net)) {
                devlink_put(devlink);
                goto next;
        }
unlock:
        rcu_read_unlock();
        return devlink;

next:
        (*indexp)++;
        goto retry;
}

/**
 * devl_register - Register devlink instance
 * @devlink: devlink
 */
int devl_register(struct devlink *devlink)
{
        ASSERT_DEVLINK_NOT_REGISTERED(devlink);
        devl_assert_locked(devlink);

        xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
        devlink_notify_register(devlink);
        devlink_rel_nested_in_notify(devlink);

        return 0;
}
EXPORT_SYMBOL_GPL(devl_register);

void devlink_register(struct devlink *devlink)
{
        devl_lock(devlink);
        devl_register(devlink);
        devl_unlock(devlink);
}
EXPORT_SYMBOL_GPL(devlink_register);

/**
 * devl_unregister - Unregister devlink instance
 * @devlink: devlink
 */
void devl_unregister(struct devlink *devlink)
{
        ASSERT_DEVLINK_REGISTERED(devlink);
        devl_assert_locked(devlink);

        devlink_notify_unregister(devlink);
        xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
        devlink_rel_put(devlink);
}
EXPORT_SYMBOL_GPL(devl_unregister);

void devlink_unregister(struct devlink *devlink)
{
        devl_lock(devlink);
        devl_unregister(devlink);
        devl_unlock(devlink);
}
EXPORT_SYMBOL_GPL(devlink_unregister);

/**
 *        devlink_alloc_ns - Allocate new devlink instance resources
 *        in specific namespace
 *
 *        @ops: ops
 *        @priv_size: size of user private data
 *        @net: net namespace
 *        @dev: parent device
 *
 *        Allocate new devlink instance resources, including devlink index
 *        and name.
 */
struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
                                 size_t priv_size, struct net *net,
                                 struct device *dev)
{
        struct devlink *devlink;
        static u32 last_id;
        int ret;

        WARN_ON(!ops || !dev);
        if (!devlink_reload_actions_valid(ops))
                return NULL;

        devlink = kvzalloc(struct_size(devlink, priv, priv_size), GFP_KERNEL);
        if (!devlink)
                return NULL;

        ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
                              &last_id, GFP_KERNEL);
        if (ret < 0)
                goto err_xa_alloc;

        devlink->dev = get_device(dev);
        devlink->ops = ops;
        xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC);
        xa_init_flags(&devlink->params, XA_FLAGS_ALLOC);
        xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
        xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC);
        write_pnet(&devlink->_net, net);
        INIT_LIST_HEAD(&devlink->rate_list);
        INIT_LIST_HEAD(&devlink->linecard_list);
        INIT_LIST_HEAD(&devlink->sb_list);
        INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
        INIT_LIST_HEAD(&devlink->resource_list);
        INIT_LIST_HEAD(&devlink->region_list);
        INIT_LIST_HEAD(&devlink->reporter_list);
        INIT_LIST_HEAD(&devlink->trap_list);
        INIT_LIST_HEAD(&devlink->trap_group_list);
        INIT_LIST_HEAD(&devlink->trap_policer_list);
        INIT_RCU_WORK(&devlink->rwork, devlink_release);
        lockdep_register_key(&devlink->lock_key);
        mutex_init(&devlink->lock);
        lockdep_set_class(&devlink->lock, &devlink->lock_key);
        refcount_set(&devlink->refcount, 1);

        return devlink;

err_xa_alloc:
        kvfree(devlink);
        return NULL;
}
EXPORT_SYMBOL_GPL(devlink_alloc_ns);

/**
 *        devlink_free - Free devlink instance resources
 *
 *        @devlink: devlink
 */
void devlink_free(struct devlink *devlink)
{
        ASSERT_DEVLINK_NOT_REGISTERED(devlink);

        WARN_ON(!list_empty(&devlink->trap_policer_list));
        WARN_ON(!list_empty(&devlink->trap_group_list));
        WARN_ON(!list_empty(&devlink->trap_list));
        WARN_ON(!list_empty(&devlink->reporter_list));
        WARN_ON(!list_empty(&devlink->region_list));
        WARN_ON(!list_empty(&devlink->resource_list));
        WARN_ON(!list_empty(&devlink->dpipe_table_list));
        WARN_ON(!list_empty(&devlink->sb_list));
        WARN_ON(!list_empty(&devlink->rate_list));
        WARN_ON(!list_empty(&devlink->linecard_list));
        WARN_ON(!xa_empty(&devlink->ports));

        xa_destroy(&devlink->nested_rels);
        xa_destroy(&devlink->snapshot_ids);
        xa_destroy(&devlink->params);
        xa_destroy(&devlink->ports);

        xa_erase(&devlinks, devlink->index);

        devlink_put(devlink);
}
EXPORT_SYMBOL_GPL(devlink_free);

static void __net_exit devlink_pernet_pre_exit(struct net *net)
{
        struct devlink *devlink;
        u32 actions_performed;
        unsigned long index;
        int err;

        /* In case network namespace is getting destroyed, reload
         * all devlink instances from this namespace into init_net.
         */
        devlinks_xa_for_each_registered_get(net, index, devlink) {
                devl_dev_lock(devlink, true);
                err = 0;
                if (devl_is_registered(devlink))
                        err = devlink_reload(devlink, &init_net,
                                             DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
                                             DEVLINK_RELOAD_LIMIT_UNSPEC,
                                             &actions_performed, NULL);
                devl_dev_unlock(devlink, true);
                devlink_put(devlink);
                if (err && err != -EOPNOTSUPP)
                        pr_warn("Failed to reload devlink instance into init_net\n");
        }
}

static struct pernet_operations devlink_pernet_ops __net_initdata = {
        .pre_exit = devlink_pernet_pre_exit,
};

static struct notifier_block devlink_port_netdevice_nb = {
        .notifier_call = devlink_port_netdevice_event,
};

static int __init devlink_init(void)
{
        int err;

        err = register_pernet_subsys(&devlink_pernet_ops);
        if (err)
                goto out;
        err = genl_register_family(&devlink_nl_family);
        if (err)
                goto out_unreg_pernet_subsys;
        err = register_netdevice_notifier(&devlink_port_netdevice_nb);
        if (!err)
                return 0;

        genl_unregister_family(&devlink_nl_family);

out_unreg_pernet_subsys:
        unregister_pernet_subsys(&devlink_pernet_ops);
out:
        WARN_ON(err);
        return err;
}

subsys_initcall(devlink_init);






















   36 





















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Definitions for the ICMP protocol.
 *
 * Version:        @(#)icmp.h        1.0.3        04/28/93
 *
 * Author:        Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 */
#ifndef _LINUX_ICMP_H
#define        _LINUX_ICMP_H

#include <linux/skbuff.h>
#include <uapi/linux/icmp.h>
#include <uapi/linux/errqueue.h>

static inline struct icmphdr *icmp_hdr(const struct sk_buff *skb)
{
        return (struct icmphdr *)skb_transport_header(skb);
}

static inline bool icmp_is_err(int type)
{
        switch (type) {
        case ICMP_DEST_UNREACH:
        case ICMP_SOURCE_QUENCH:
        case ICMP_REDIRECT:
        case ICMP_TIME_EXCEEDED:
        case ICMP_PARAMETERPROB:
                return true;
        }

        return false;
}

void ip_icmp_error_rfc4884(const struct sk_buff *skb,
                           struct sock_ee_data_rfc4884 *out,
                           int thlen, int off);

#endif        /* _LINUX_ICMP_H */












































































   12 






































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* Filesystem parameter description and parser
 *
 * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#ifndef _LINUX_FS_PARSER_H
#define _LINUX_FS_PARSER_H

#include <linux/fs_context.h>

struct path;

struct constant_table {
        const char        *name;
        int                value;
};

struct fs_parameter_spec;
struct fs_parse_result;
typedef int fs_param_type(struct p_log *,
                          const struct fs_parameter_spec *,
                          struct fs_parameter *,
                          struct fs_parse_result *);
/*
 * The type of parameter expected.
 */
fs_param_type fs_param_is_bool, fs_param_is_u32, fs_param_is_s32, fs_param_is_u64,
        fs_param_is_enum, fs_param_is_string, fs_param_is_blob, fs_param_is_blockdev,
        fs_param_is_path, fs_param_is_fd, fs_param_is_uid, fs_param_is_gid,
        fs_param_is_file_or_string;

/*
 * Specification of the type of value a parameter wants.
 *
 * Note that the fsparam_flag(), fsparam_string(), fsparam_u32(), ... macros
 * should be used to generate elements of this type.
 */
struct fs_parameter_spec {
        const char                *name;
        fs_param_type                *type;        /* The desired parameter type */
        u8                        opt;        /* Option number (returned by fs_parse()) */
        unsigned short                flags;
#define fs_param_neg_with_no        0x0002        /* "noxxx" is negative param */
#define fs_param_can_be_empty        0x0004        /* "xxx=" is allowed */
#define fs_param_deprecated        0x0008        /* The param is deprecated */
        const void                *data;
};

/*
 * Result of parse.
 */
struct fs_parse_result {
        bool                        negated;        /* T if param was "noxxx" */
        union {
                bool                boolean;        /* For spec_bool */
                int                int_32;                /* For spec_s32/spec_enum */
                unsigned int        uint_32;        /* For spec_u32{,_octal,_hex}/spec_enum */
                u64                uint_64;        /* For spec_u64 */
                kuid_t                uid;
                kgid_t                gid;
        };
};

extern int __fs_parse(struct p_log *log,
                    const struct fs_parameter_spec *desc,
                    struct fs_parameter *value,
                    struct fs_parse_result *result);

static inline int fs_parse(struct fs_context *fc,
             const struct fs_parameter_spec *desc,
             struct fs_parameter *param,
             struct fs_parse_result *result)
{
        return __fs_parse(&fc->log, desc, param, result);
}

extern int fs_lookup_param(struct fs_context *fc,
                           struct fs_parameter *param,
                           bool want_bdev,
                           unsigned int flags,
                           struct path *_path);

extern int lookup_constant(const struct constant_table tbl[], const char *name, int not_found);

#ifdef CONFIG_VALIDATE_FS_PARSER
extern bool validate_constant_table(const struct constant_table *tbl, size_t tbl_size,
                                    int low, int high, int special);
extern bool fs_validate_description(const char *name,
                                    const struct fs_parameter_spec *desc);
#else
static inline bool validate_constant_table(const struct constant_table *tbl, size_t tbl_size,
                                           int low, int high, int special)
{ return true; }
static inline bool fs_validate_description(const char *name,
                                           const struct fs_parameter_spec *desc)
{ return true; }
#endif

/*
 * Parameter type, name, index and flags element constructors.  Use as:
 *
 *  fsparam_xxxx("foo", Opt_foo)
 *
 * If existing helpers are not enough, direct use of __fsparam() would
 * work, but any such case is probably a sign that new helper is needed.
 * Helpers will remain stable; low-level implementation may change.
 */
#define __fsparam(TYPE, NAME, OPT, FLAGS, DATA) \
        { \
                .name = NAME, \
                .opt = OPT, \
                .type = TYPE, \
                .flags = FLAGS, \
                .data = DATA \
        }

#define fsparam_flag(NAME, OPT)        __fsparam(NULL, NAME, OPT, 0, NULL)
#define fsparam_flag_no(NAME, OPT) \
                        __fsparam(NULL, NAME, OPT, fs_param_neg_with_no, NULL)
#define fsparam_bool(NAME, OPT)        __fsparam(fs_param_is_bool, NAME, OPT, 0, NULL)
#define fsparam_u32(NAME, OPT)        __fsparam(fs_param_is_u32, NAME, OPT, 0, NULL)
#define fsparam_u32oct(NAME, OPT) \
                        __fsparam(fs_param_is_u32, NAME, OPT, 0, (void *)8)
#define fsparam_u32hex(NAME, OPT) \
                        __fsparam(fs_param_is_u32_hex, NAME, OPT, 0, (void *)16)
#define fsparam_s32(NAME, OPT)        __fsparam(fs_param_is_s32, NAME, OPT, 0, NULL)
#define fsparam_u64(NAME, OPT)        __fsparam(fs_param_is_u64, NAME, OPT, 0, NULL)
#define fsparam_enum(NAME, OPT, array)        __fsparam(fs_param_is_enum, NAME, OPT, 0, array)
#define fsparam_string(NAME, OPT) \
                                __fsparam(fs_param_is_string, NAME, OPT, 0, NULL)
#define fsparam_blob(NAME, OPT)        __fsparam(fs_param_is_blob, NAME, OPT, 0, NULL)
#define fsparam_bdev(NAME, OPT)        __fsparam(fs_param_is_blockdev, NAME, OPT, 0, NULL)
#define fsparam_path(NAME, OPT)        __fsparam(fs_param_is_path, NAME, OPT, 0, NULL)
#define fsparam_fd(NAME, OPT)        __fsparam(fs_param_is_fd, NAME, OPT, 0, NULL)
#define fsparam_file_or_string(NAME, OPT) \
                                __fsparam(fs_param_is_file_or_string, NAME, OPT, 0, NULL)
#define fsparam_uid(NAME, OPT) __fsparam(fs_param_is_uid, NAME, OPT, 0, NULL)
#define fsparam_gid(NAME, OPT) __fsparam(fs_param_is_gid, NAME, OPT, 0, NULL)

/* String parameter that allows empty argument */
#define fsparam_string_empty(NAME, OPT) \
        __fsparam(fs_param_is_string, NAME, OPT, fs_param_can_be_empty, NULL)

#endif /* _LINUX_FS_PARSER_H */
































































































































































































































































































































































































































































































 9081 















































  150 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_PREEMPT_H
#define __LINUX_PREEMPT_H

/*
 * include/linux/preempt.h - macros for accessing and manipulating
 * preempt_count (used for kernel preemption, interrupt count, etc.)
 */

#include <linux/linkage.h>
#include <linux/cleanup.h>
#include <linux/types.h>

/*
 * We put the hardirq and softirq counter into the preemption
 * counter. The bitmask has the following meaning:
 *
 * - bits 0-7 are the preemption count (max preemption depth: 256)
 * - bits 8-15 are the softirq count (max # of softirqs: 256)
 *
 * The hardirq count could in theory be the same as the number of
 * interrupts in the system, but we run all interrupt handlers with
 * interrupts disabled, so we cannot have nesting interrupts. Though
 * there are a few palaeontologic drivers which reenable interrupts in
 * the handler, so we need more than one bit here.
 *
 *         PREEMPT_MASK:        0x000000ff
 *         SOFTIRQ_MASK:        0x0000ff00
 *         HARDIRQ_MASK:        0x000f0000
 *             NMI_MASK:        0x00f00000
 * PREEMPT_NEED_RESCHED:        0x80000000
 */
#define PREEMPT_BITS        8
#define SOFTIRQ_BITS        8
#define HARDIRQ_BITS        4
#define NMI_BITS        4

#define PREEMPT_SHIFT        0
#define SOFTIRQ_SHIFT        (PREEMPT_SHIFT + PREEMPT_BITS)
#define HARDIRQ_SHIFT        (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
#define NMI_SHIFT        (HARDIRQ_SHIFT + HARDIRQ_BITS)

#define __IRQ_MASK(x)        ((1UL << (x))-1)

#define PREEMPT_MASK        (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
#define SOFTIRQ_MASK        (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
#define HARDIRQ_MASK        (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
#define NMI_MASK        (__IRQ_MASK(NMI_BITS)     << NMI_SHIFT)

#define PREEMPT_OFFSET        (1UL << PREEMPT_SHIFT)
#define SOFTIRQ_OFFSET        (1UL << SOFTIRQ_SHIFT)
#define HARDIRQ_OFFSET        (1UL << HARDIRQ_SHIFT)
#define NMI_OFFSET        (1UL << NMI_SHIFT)

#define SOFTIRQ_DISABLE_OFFSET        (2 * SOFTIRQ_OFFSET)

#define PREEMPT_DISABLED        (PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)

/*
 * Disable preemption until the scheduler is running -- use an unconditional
 * value so that it also works on !PREEMPT_COUNT kernels.
 *
 * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count().
 */
#define INIT_PREEMPT_COUNT        PREEMPT_OFFSET

/*
 * Initial preempt_count value; reflects the preempt_count schedule invariant
 * which states that during context switches:
 *
 *    preempt_count() == 2*PREEMPT_DISABLE_OFFSET
 *
 * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels.
 * Note: See finish_task_switch().
 */
#define FORK_PREEMPT_COUNT        (2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)

/* preempt_count() and related functions, depends on PREEMPT_NEED_RESCHED */
#include <asm/preempt.h>

/**
 * interrupt_context_level - return interrupt context level
 *
 * Returns the current interrupt context level.
 *  0 - normal context
 *  1 - softirq context
 *  2 - hardirq context
 *  3 - NMI context
 */
static __always_inline unsigned char interrupt_context_level(void)
{
        unsigned long pc = preempt_count();
        unsigned char level = 0;

        level += !!(pc & (NMI_MASK));
        level += !!(pc & (NMI_MASK | HARDIRQ_MASK));
        level += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));

        return level;
}

/*
 * These macro definitions avoid redundant invocations of preempt_count()
 * because such invocations would result in redundant loads given that
 * preempt_count() is commonly implemented with READ_ONCE().
 */

#define nmi_count()        (preempt_count() & NMI_MASK)
#define hardirq_count()        (preempt_count() & HARDIRQ_MASK)
#ifdef CONFIG_PREEMPT_RT
# define softirq_count()        (current->softirq_disable_cnt & SOFTIRQ_MASK)
# define irq_count()                ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count())
#else
# define softirq_count()        (preempt_count() & SOFTIRQ_MASK)
# define irq_count()                (preempt_count() & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_MASK))
#endif

/*
 * Macros to retrieve the current execution context:
 *
 * in_nmi()                - We're in NMI context
 * in_hardirq()                - We're in hard IRQ context
 * in_serving_softirq()        - We're in softirq context
 * in_task()                - We're in task context
 */
#define in_nmi()                (nmi_count())
#define in_hardirq()                (hardirq_count())
#define in_serving_softirq()        (softirq_count() & SOFTIRQ_OFFSET)
#ifdef CONFIG_PREEMPT_RT
# define in_task()                (!((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | in_serving_softirq()))
#else
# define in_task()                (!(preempt_count() & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)))
#endif

/*
 * The following macros are deprecated and should not be used in new code:
 * in_irq()       - Obsolete version of in_hardirq()
 * in_softirq()   - We have BH disabled, or are processing softirqs
 * in_interrupt() - We're in NMI,IRQ,SoftIRQ context or have BH disabled
 */
#define in_irq()                (hardirq_count())
#define in_softirq()                (softirq_count())
#define in_interrupt()                (irq_count())

/*
 * The preempt_count offset after preempt_disable();
 */
#if defined(CONFIG_PREEMPT_COUNT)
# define PREEMPT_DISABLE_OFFSET        PREEMPT_OFFSET
#else
# define PREEMPT_DISABLE_OFFSET        0
#endif

/*
 * The preempt_count offset after spin_lock()
 */
#if !defined(CONFIG_PREEMPT_RT)
#define PREEMPT_LOCK_OFFSET                PREEMPT_DISABLE_OFFSET
#else
/* Locks on RT do not disable preemption */
#define PREEMPT_LOCK_OFFSET                0
#endif

/*
 * The preempt_count offset needed for things like:
 *
 *  spin_lock_bh()
 *
 * Which need to disable both preemption (CONFIG_PREEMPT_COUNT) and
 * softirqs, such that unlock sequences of:
 *
 *  spin_unlock();
 *  local_bh_enable();
 *
 * Work as expected.
 */
#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_LOCK_OFFSET)

/*
 * Are we running in atomic context?  WARNING: this macro cannot
 * always detect atomic context; in particular, it cannot know about
 * held spinlocks in non-preemptible kernels.  Thus it should not be
 * used in the general case to determine whether sleeping is possible.
 * Do not use in_atomic() in driver code.
 */
#define in_atomic()        (preempt_count() != 0)

/*
 * Check whether we were atomic before we did preempt_disable():
 * (used by the scheduler)
 */
#define in_atomic_preempt_off() (preempt_count() != PREEMPT_DISABLE_OFFSET)

#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE)
extern void preempt_count_add(int val);
extern void preempt_count_sub(int val);
#define preempt_count_dec_and_test() \
        ({ preempt_count_sub(1); should_resched(0); })
#else
#define preempt_count_add(val)        __preempt_count_add(val)
#define preempt_count_sub(val)        __preempt_count_sub(val)
#define preempt_count_dec_and_test() __preempt_count_dec_and_test()
#endif

#define __preempt_count_inc() __preempt_count_add(1)
#define __preempt_count_dec() __preempt_count_sub(1)

#define preempt_count_inc() preempt_count_add(1)
#define preempt_count_dec() preempt_count_sub(1)

#ifdef CONFIG_PREEMPT_COUNT

#define preempt_disable() \
do { \
        preempt_count_inc(); \
        barrier(); \
} while (0)

#define sched_preempt_enable_no_resched() \
do { \
        barrier(); \
        preempt_count_dec(); \
} while (0)

#define preempt_enable_no_resched() sched_preempt_enable_no_resched()

#define preemptible()        (preempt_count() == 0 && !irqs_disabled())

#ifdef CONFIG_PREEMPTION
#define preempt_enable() \
do { \
        barrier(); \
        if (unlikely(preempt_count_dec_and_test())) \
                __preempt_schedule(); \
} while (0)

#define preempt_enable_notrace() \
do { \
        barrier(); \
        if (unlikely(__preempt_count_dec_and_test())) \
                __preempt_schedule_notrace(); \
} while (0)

#define preempt_check_resched() \
do { \
        if (should_resched(0)) \
                __preempt_schedule(); \
} while (0)

#else /* !CONFIG_PREEMPTION */
#define preempt_enable() \
do { \
        barrier(); \
        preempt_count_dec(); \
} while (0)

#define preempt_enable_notrace() \
do { \
        barrier(); \
        __preempt_count_dec(); \
} while (0)

#define preempt_check_resched() do { } while (0)
#endif /* CONFIG_PREEMPTION */

#define preempt_disable_notrace() \
do { \
        __preempt_count_inc(); \
        barrier(); \
} while (0)

#define preempt_enable_no_resched_notrace() \
do { \
        barrier(); \
        __preempt_count_dec(); \
} while (0)

#else /* !CONFIG_PREEMPT_COUNT */

/*
 * Even if we don't have any preemption, we need preempt disable/enable
 * to be barriers, so that we don't have things like get_user/put_user
 * that can cause faults and scheduling migrate into our preempt-protected
 * region.
 */
#define preempt_disable()                        barrier()
#define sched_preempt_enable_no_resched()        barrier()
#define preempt_enable_no_resched()                barrier()
#define preempt_enable()                        barrier()
#define preempt_check_resched()                        do { } while (0)

#define preempt_disable_notrace()                barrier()
#define preempt_enable_no_resched_notrace()        barrier()
#define preempt_enable_notrace()                barrier()
#define preemptible()                                0

#endif /* CONFIG_PREEMPT_COUNT */

#ifdef MODULE
/*
 * Modules have no business playing preemption tricks.
 */
#undef sched_preempt_enable_no_resched
#undef preempt_enable_no_resched
#undef preempt_enable_no_resched_notrace
#undef preempt_check_resched
#endif

#define preempt_set_need_resched() \
do { \
        set_preempt_need_resched(); \
} while (0)
#define preempt_fold_need_resched() \
do { \
        if (tif_need_resched()) \
                set_preempt_need_resched(); \
} while (0)

#ifdef CONFIG_PREEMPT_NOTIFIERS

struct preempt_notifier;

/**
 * preempt_ops - notifiers called when a task is preempted and rescheduled
 * @sched_in: we're about to be rescheduled:
 *    notifier: struct preempt_notifier for the task being scheduled
 *    cpu:  cpu we're scheduled on
 * @sched_out: we've just been preempted
 *    notifier: struct preempt_notifier for the task being preempted
 *    next: the task that's kicking us out
 *
 * Please note that sched_in and out are called under different
 * contexts.  sched_out is called with rq lock held and irq disabled
 * while sched_in is called without rq lock and irq enabled.  This
 * difference is intentional and depended upon by its users.
 */
struct preempt_ops {
        void (*sched_in)(struct preempt_notifier *notifier, int cpu);
        void (*sched_out)(struct preempt_notifier *notifier,
                          struct task_struct *next);
};

/**
 * preempt_notifier - key for installing preemption notifiers
 * @link: internal use
 * @ops: defines the notifier functions to be called
 *
 * Usually used in conjunction with container_of().
 */
struct preempt_notifier {
        struct hlist_node link;
        struct preempt_ops *ops;
};

void preempt_notifier_inc(void);
void preempt_notifier_dec(void);
void preempt_notifier_register(struct preempt_notifier *notifier);
void preempt_notifier_unregister(struct preempt_notifier *notifier);

static inline void preempt_notifier_init(struct preempt_notifier *notifier,
                                     struct preempt_ops *ops)
{
        /* INIT_HLIST_NODE() open coded, to avoid dependency on list.h */
        notifier->link.next = NULL;
        notifier->link.pprev = NULL;
        notifier->ops = ops;
}

#endif

#ifdef CONFIG_SMP

/*
 * Migrate-Disable and why it is undesired.
 *
 * When a preempted task becomes elegible to run under the ideal model (IOW it
 * becomes one of the M highest priority tasks), it might still have to wait
 * for the preemptee's migrate_disable() section to complete. Thereby suffering
 * a reduction in bandwidth in the exact duration of the migrate_disable()
 * section.
 *
 * Per this argument, the change from preempt_disable() to migrate_disable()
 * gets us:
 *
 * - a higher priority tasks gains reduced wake-up latency; with preempt_disable()
 *   it would have had to wait for the lower priority task.
 *
 * - a lower priority tasks; which under preempt_disable() could've instantly
 *   migrated away when another CPU becomes available, is now constrained
 *   by the ability to push the higher priority task away, which might itself be
 *   in a migrate_disable() section, reducing it's available bandwidth.
 *
 * IOW it trades latency / moves the interference term, but it stays in the
 * system, and as long as it remains unbounded, the system is not fully
 * deterministic.
 *
 *
 * The reason we have it anyway.
 *
 * PREEMPT_RT breaks a number of assumptions traditionally held. By forcing a
 * number of primitives into becoming preemptible, they would also allow
 * migration. This turns out to break a bunch of per-cpu usage. To this end,
 * all these primitives employ migirate_disable() to restore this implicit
 * assumption.
 *
 * This is a 'temporary' work-around at best. The correct solution is getting
 * rid of the above assumptions and reworking the code to employ explicit
 * per-cpu locking or short preempt-disable regions.
 *
 * The end goal must be to get rid of migrate_disable(), alternatively we need
 * a schedulability theory that does not depend on abritrary migration.
 *
 *
 * Notes on the implementation.
 *
 * The implementation is particularly tricky since existing code patterns
 * dictate neither migrate_disable() nor migrate_enable() is allowed to block.
 * This means that it cannot use cpus_read_lock() to serialize against hotplug,
 * nor can it easily migrate itself into a pending affinity mask change on
 * migrate_enable().
 *
 *
 * Note: even non-work-conserving schedulers like semi-partitioned depends on
 *       migration, so migrate_disable() is not only a problem for
 *       work-conserving schedulers.
 *
 */
extern void migrate_disable(void);
extern void migrate_enable(void);

#else

static inline void migrate_disable(void) { }
static inline void migrate_enable(void) { }

#endif /* CONFIG_SMP */

/**
 * preempt_disable_nested - Disable preemption inside a normally preempt disabled section
 *
 * Use for code which requires preemption protection inside a critical
 * section which has preemption disabled implicitly on non-PREEMPT_RT
 * enabled kernels, by e.g.:
 *  - holding a spinlock/rwlock
 *  - soft interrupt context
 *  - regular interrupt handlers
 *
 * On PREEMPT_RT enabled kernels spinlock/rwlock held sections, soft
 * interrupt context and regular interrupt handlers are preemptible and
 * only prevent migration. preempt_disable_nested() ensures that preemption
 * is disabled for cases which require CPU local serialization even on
 * PREEMPT_RT. For non-PREEMPT_RT kernels this is a NOP.
 *
 * The use cases are code sequences which are not serialized by a
 * particular lock instance, e.g.:
 *  - seqcount write side critical sections where the seqcount is not
 *    associated to a particular lock and therefore the automatic
 *    protection mechanism does not work. This prevents a live lock
 *    against a preempting high priority reader.
 *  - RMW per CPU variable updates like vmstat.
 */
/* Macro to avoid header recursion hell vs. lockdep */
#define preempt_disable_nested()                                \
do {                                                                \
        if (IS_ENABLED(CONFIG_PREEMPT_RT))                        \
                preempt_disable();                                \
        else                                                        \
                lockdep_assert_preemption_disabled();                \
} while (0)

/**
 * preempt_enable_nested - Undo the effect of preempt_disable_nested()
 */
static __always_inline void preempt_enable_nested(void)
{
        if (IS_ENABLED(CONFIG_PREEMPT_RT))
                preempt_enable();
}

DEFINE_LOCK_GUARD_0(preempt, preempt_disable(), preempt_enable())
DEFINE_LOCK_GUARD_0(preempt_notrace, preempt_disable_notrace(), preempt_enable_notrace())
DEFINE_LOCK_GUARD_0(migrate, migrate_disable(), migrate_enable())

#ifdef CONFIG_PREEMPT_DYNAMIC

extern bool preempt_model_none(void);
extern bool preempt_model_voluntary(void);
extern bool preempt_model_full(void);
extern bool preempt_model_lazy(void);

#else

static inline bool preempt_model_none(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_NONE);
}
static inline bool preempt_model_voluntary(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY);
}
static inline bool preempt_model_full(void)
{
        return IS_ENABLED(CONFIG_PREEMPT);
}

static inline bool preempt_model_lazy(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_LAZY);
}

#endif

static inline bool preempt_model_rt(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_RT);
}

/*
 * Does the preemption model allow non-cooperative preemption?
 *
 * For !CONFIG_PREEMPT_DYNAMIC kernels this is an exact match with
 * CONFIG_PREEMPTION; for CONFIG_PREEMPT_DYNAMIC this doesn't work as the
 * kernel is *built* with CONFIG_PREEMPTION=y but may run with e.g. the
 * PREEMPT_NONE model.
 */
static inline bool preempt_model_preemptible(void)
{
        return preempt_model_full() || preempt_model_lazy() || preempt_model_rt();
}

#endif /* __LINUX_PREEMPT_H */



































































































    3 
















    3 





    3 






    3 
    3 
    3 
    3 

    3 



    3 
























































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
// SPDX-License-Identifier: GPL-2.0
/*
 * devtmpfs - kernel-maintained tmpfs-based /dev
 *
 * Copyright (C) 2009, Kay Sievers <kay.sievers@vrfy.org>
 *
 * During bootup, before any driver core device is registered,
 * devtmpfs, a tmpfs-based filesystem is created. Every driver-core
 * device which requests a device node, will add a node in this
 * filesystem.
 * By default, all devices are named after the name of the device,
 * owned by root and have a default mode of 0600. Subsystems can
 * overwrite the default setting if needed.
 */

#define pr_fmt(fmt) "devtmpfs: " fmt

#include <linux/kernel.h>
#include <linux/syscalls.h>
#include <linux/mount.h>
#include <linux/device.h>
#include <linux/blkdev.h>
#include <linux/namei.h>
#include <linux/fs.h>
#include <linux/shmem_fs.h>
#include <linux/ramfs.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/kthread.h>
#include <linux/init_syscalls.h>
#include <uapi/linux/mount.h>
#include "base.h"

#ifdef CONFIG_DEVTMPFS_SAFE
#define DEVTMPFS_MFLAGS       (MS_SILENT | MS_NOEXEC | MS_NOSUID)
#else
#define DEVTMPFS_MFLAGS       (MS_SILENT)
#endif

static struct task_struct *thread;

static int __initdata mount_dev = IS_ENABLED(CONFIG_DEVTMPFS_MOUNT);

static DEFINE_SPINLOCK(req_lock);

static struct req {
        struct req *next;
        struct completion done;
        int err;
        const char *name;
        umode_t mode;        /* 0 => delete */
        kuid_t uid;
        kgid_t gid;
        struct device *dev;
} *requests;

static int __init mount_param(char *str)
{
        mount_dev = simple_strtoul(str, NULL, 0);
        return 1;
}
__setup("devtmpfs.mount=", mount_param);

static struct vfsmount *mnt;

static struct dentry *public_dev_mount(struct file_system_type *fs_type, int flags,
                      const char *dev_name, void *data)
{
        struct super_block *s = mnt->mnt_sb;
        int err;

        atomic_inc(&s->s_active);
        down_write(&s->s_umount);
        err = reconfigure_single(s, flags, data);
        if (err < 0) {
                deactivate_locked_super(s);
                return ERR_PTR(err);
        }
        return dget(s->s_root);
}

static struct file_system_type internal_fs_type = {
        .name = "devtmpfs",
#ifdef CONFIG_TMPFS
        .init_fs_context = shmem_init_fs_context,
#else
        .init_fs_context = ramfs_init_fs_context,
#endif
        .kill_sb = kill_litter_super,
};

static struct file_system_type dev_fs_type = {
        .name = "devtmpfs",
        .mount = public_dev_mount,
};

static int devtmpfs_submit_req(struct req *req, const char *tmp)
{
        init_completion(&req->done);

        spin_lock(&req_lock);
        req->next = requests;
        requests = req;
        spin_unlock(&req_lock);

        wake_up_process(thread);
        wait_for_completion(&req->done);

        kfree(tmp);

        return req->err;
}

int devtmpfs_create_node(struct device *dev)
{
        const char *tmp = NULL;
        struct req req;

        if (!thread)
                return 0;

        req.mode = 0;
        req.uid = GLOBAL_ROOT_UID;
        req.gid = GLOBAL_ROOT_GID;
        req.name = device_get_devnode(dev, &req.mode, &req.uid, &req.gid, &tmp);
        if (!req.name)
                return -ENOMEM;

        if (req.mode == 0)
                req.mode = 0600;
        if (is_blockdev(dev))
                req.mode |= S_IFBLK;
        else
                req.mode |= S_IFCHR;

        req.dev = dev;

        return devtmpfs_submit_req(&req, tmp);
}

int devtmpfs_delete_node(struct device *dev)
{
        const char *tmp = NULL;
        struct req req;

        if (!thread)
                return 0;

        req.name = device_get_devnode(dev, NULL, NULL, NULL, &tmp);
        if (!req.name)
                return -ENOMEM;

        req.mode = 0;
        req.dev = dev;

        return devtmpfs_submit_req(&req, tmp);
}

static int dev_mkdir(const char *name, umode_t mode)
{
        struct dentry *dentry;
        struct path path;
        int err;

        dentry = kern_path_create(AT_FDCWD, name, &path, LOOKUP_DIRECTORY);
        if (IS_ERR(dentry))
                return PTR_ERR(dentry);

        err = vfs_mkdir(&nop_mnt_idmap, d_inode(path.dentry), dentry, mode);
        if (!err)
                /* mark as kernel-created inode */
                d_inode(dentry)->i_private = &thread;
        done_path_create(&path, dentry);
        return err;
}

static int create_path(const char *nodepath)
{
        char *path;
        char *s;
        int err = 0;

        /* parent directories do not exist, create them */
        path = kstrdup(nodepath, GFP_KERNEL);
        if (!path)
                return -ENOMEM;

        s = path;
        for (;;) {
                s = strchr(s, '/');
                if (!s)
                        break;
                s[0] = '\0';
                err = dev_mkdir(path, 0755);
                if (err && err != -EEXIST)
                        break;
                s[0] = '/';
                s++;
        }
        kfree(path);
        return err;
}

static int handle_create(const char *nodename, umode_t mode, kuid_t uid,
                         kgid_t gid, struct device *dev)
{
        struct dentry *dentry;
        struct path path;
        int err;

        dentry = kern_path_create(AT_FDCWD, nodename, &path, 0);
        if (dentry == ERR_PTR(-ENOENT)) {
                create_path(nodename);
                dentry = kern_path_create(AT_FDCWD, nodename, &path, 0);
        }
        if (IS_ERR(dentry))
                return PTR_ERR(dentry);

        err = vfs_mknod(&nop_mnt_idmap, d_inode(path.dentry), dentry, mode,
                        dev->devt);
        if (!err) {
                struct iattr newattrs;

                newattrs.ia_mode = mode;
                newattrs.ia_uid = uid;
                newattrs.ia_gid = gid;
                newattrs.ia_valid = ATTR_MODE|ATTR_UID|ATTR_GID;
                inode_lock(d_inode(dentry));
                notify_change(&nop_mnt_idmap, dentry, &newattrs, NULL);
                inode_unlock(d_inode(dentry));

                /* mark as kernel-created inode */
                d_inode(dentry)->i_private = &thread;
        }
        done_path_create(&path, dentry);
        return err;
}

static int dev_rmdir(const char *name)
{
        struct path parent;
        struct dentry *dentry;
        int err;

        dentry = kern_path_locked(name, &parent);
        if (IS_ERR(dentry))
                return PTR_ERR(dentry);
        if (d_really_is_positive(dentry)) {
                if (d_inode(dentry)->i_private == &thread)
                        err = vfs_rmdir(&nop_mnt_idmap, d_inode(parent.dentry),
                                        dentry);
                else
                        err = -EPERM;
        } else {
                err = -ENOENT;
        }
        dput(dentry);
        inode_unlock(d_inode(parent.dentry));
        path_put(&parent);
        return err;
}

static int delete_path(const char *nodepath)
{
        char *path;
        int err = 0;

        path = kstrdup(nodepath, GFP_KERNEL);
        if (!path)
                return -ENOMEM;

        for (;;) {
                char *base;

                base = strrchr(path, '/');
                if (!base)
                        break;
                base[0] = '\0';
                err = dev_rmdir(path);
                if (err)
                        break;
        }

        kfree(path);
        return err;
}

static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *stat)
{
        /* did we create it */
        if (inode->i_private != &thread)
                return 0;

        /* does the dev_t match */
        if (is_blockdev(dev)) {
                if (!S_ISBLK(stat->mode))
                        return 0;
        } else {
                if (!S_ISCHR(stat->mode))
                        return 0;
        }
        if (stat->rdev != dev->devt)
                return 0;

        /* ours */
        return 1;
}

static int handle_remove(const char *nodename, struct device *dev)
{
        struct path parent;
        struct dentry *dentry;
        int deleted = 0;
        int err;

        dentry = kern_path_locked(nodename, &parent);
        if (IS_ERR(dentry))
                return PTR_ERR(dentry);

        if (d_really_is_positive(dentry)) {
                struct kstat stat;
                struct path p = {.mnt = parent.mnt, .dentry = dentry};
                err = vfs_getattr(&p, &stat, STATX_TYPE | STATX_MODE,
                                  AT_STATX_SYNC_AS_STAT);
                if (!err && dev_mynode(dev, d_inode(dentry), &stat)) {
                        struct iattr newattrs;
                        /*
                         * before unlinking this node, reset permissions
                         * of possible references like hardlinks
                         */
                        newattrs.ia_uid = GLOBAL_ROOT_UID;
                        newattrs.ia_gid = GLOBAL_ROOT_GID;
                        newattrs.ia_mode = stat.mode & ~0777;
                        newattrs.ia_valid =
                                ATTR_UID|ATTR_GID|ATTR_MODE;
                        inode_lock(d_inode(dentry));
                        notify_change(&nop_mnt_idmap, dentry, &newattrs, NULL);
                        inode_unlock(d_inode(dentry));
                        err = vfs_unlink(&nop_mnt_idmap, d_inode(parent.dentry),
                                         dentry, NULL);
                        if (!err || err == -ENOENT)
                                deleted = 1;
                }
        } else {
                err = -ENOENT;
        }
        dput(dentry);
        inode_unlock(d_inode(parent.dentry));

        path_put(&parent);
        if (deleted && strchr(nodename, '/'))
                delete_path(nodename);
        return err;
}

/*
 * If configured, or requested by the commandline, devtmpfs will be
 * auto-mounted after the kernel mounted the root filesystem.
 */
int __init devtmpfs_mount(void)
{
        int err;

        if (!mount_dev)
                return 0;

        if (!thread)
                return 0;

        err = init_mount("devtmpfs", "dev", "devtmpfs", DEVTMPFS_MFLAGS, NULL);
        if (err)
                pr_info("error mounting %d\n", err);
        else
                pr_info("mounted\n");
        return err;
}

static __initdata DECLARE_COMPLETION(setup_done);

static int handle(const char *name, umode_t mode, kuid_t uid, kgid_t gid,
                  struct device *dev)
{
        if (mode)
                return handle_create(name, mode, uid, gid, dev);
        else
                return handle_remove(name, dev);
}

static void __noreturn devtmpfs_work_loop(void)
{
        while (1) {
                spin_lock(&req_lock);
                while (requests) {
                        struct req *req = requests;
                        requests = NULL;
                        spin_unlock(&req_lock);
                        while (req) {
                                struct req *next = req->next;
                                req->err = handle(req->name, req->mode,
                                                  req->uid, req->gid, req->dev);
                                complete(&req->done);
                                req = next;
                        }
                        spin_lock(&req_lock);
                }
                __set_current_state(TASK_INTERRUPTIBLE);
                spin_unlock(&req_lock);
                schedule();
        }
}

static noinline int __init devtmpfs_setup(void *p)
{
        int err;

        err = ksys_unshare(CLONE_NEWNS);
        if (err)
                goto out;
        err = init_mount("devtmpfs", "/", "devtmpfs", DEVTMPFS_MFLAGS, NULL);
        if (err)
                goto out;
        init_chdir("/.."); /* will traverse into overmounted root */
        init_chroot(".");
out:
        *(int *)p = err;
        return err;
}

/*
 * The __ref is because devtmpfs_setup needs to be __init for the routines it
 * calls.  That call is done while devtmpfs_init, which is marked __init,
 * synchronously waits for it to complete.
 */
static int __ref devtmpfsd(void *p)
{
        int err = devtmpfs_setup(p);

        complete(&setup_done);
        if (err)
                return err;
        devtmpfs_work_loop();
        return 0;
}

/*
 * Create devtmpfs instance, driver-core devices will add their device
 * nodes here.
 */
int __init devtmpfs_init(void)
{
        char opts[] = "mode=0755";
        int err;

        mnt = vfs_kern_mount(&internal_fs_type, 0, "devtmpfs", opts);
        if (IS_ERR(mnt)) {
                pr_err("unable to create devtmpfs %ld\n", PTR_ERR(mnt));
                return PTR_ERR(mnt);
        }
        err = register_filesystem(&dev_fs_type);
        if (err) {
                pr_err("unable to register devtmpfs type %d\n", err);
                return err;
        }

        thread = kthread_run(devtmpfsd, &err, "kdevtmpfs");
        if (!IS_ERR(thread)) {
                wait_for_completion(&setup_done);
        } else {
                err = PTR_ERR(thread);
                thread = NULL;
        }

        if (err) {
                pr_err("unable to create devtmpfs %d\n", err);
                unregister_filesystem(&dev_fs_type);
                thread = NULL;
                return err;
        }

        pr_info("initialized\n");
        return 0;
}























































































































































    3 

    3 
    3 


    3 




























    5 




    5 
    2 





    1 




























    4 
    4 
    2 














    2 



    2 
    2 


    2 
    2 




    2 
    2 


    2 




    2 
    2 


    2 










    4 









    1 
    1 





    4 



    3 



    1 

    2 


    2 
    2 
    2 


    2 
    2 



    2 

















    2 



    1 






    1 















    2 
    2 

    2 


    2 


    2 















    2 

    2 
    2 





    1 












    1 

    3 

    3 
    3 


    3 
    3 



    3 


    2 
    2 


    2 


    3 



    1 













    3 
    3 


    3 
    3 
    3 
    3 





    3 
    3 


    1 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
// SPDX-License-Identifier: GPL-2.0-only

#include <linux/ethtool.h>
#include <linux/phy.h>
#include "netlink.h"
#include "common.h"

struct strset_info {
        bool per_dev;
        bool free_strings;
        unsigned int count;
        const char (*strings)[ETH_GSTRING_LEN];
};

static const struct strset_info info_template[] = {
        [ETH_SS_TEST] = {
                .per_dev        = true,
        },
        [ETH_SS_STATS] = {
                .per_dev        = true,
        },
        [ETH_SS_PRIV_FLAGS] = {
                .per_dev        = true,
        },
        [ETH_SS_FEATURES] = {
                .per_dev        = false,
                .count                = ARRAY_SIZE(netdev_features_strings),
                .strings        = netdev_features_strings,
        },
        [ETH_SS_RSS_HASH_FUNCS] = {
                .per_dev        = false,
                .count                = ARRAY_SIZE(rss_hash_func_strings),
                .strings        = rss_hash_func_strings,
        },
        [ETH_SS_TUNABLES] = {
                .per_dev        = false,
                .count                = ARRAY_SIZE(tunable_strings),
                .strings        = tunable_strings,
        },
        [ETH_SS_PHY_STATS] = {
                .per_dev        = true,
        },
        [ETH_SS_PHY_TUNABLES] = {
                .per_dev        = false,
                .count                = ARRAY_SIZE(phy_tunable_strings),
                .strings        = phy_tunable_strings,
        },
        [ETH_SS_LINK_MODES] = {
                .per_dev        = false,
                .count                = __ETHTOOL_LINK_MODE_MASK_NBITS,
                .strings        = link_mode_names,
        },
        [ETH_SS_MSG_CLASSES] = {
                .per_dev        = false,
                .count                = NETIF_MSG_CLASS_COUNT,
                .strings        = netif_msg_class_names,
        },
        [ETH_SS_WOL_MODES] = {
                .per_dev        = false,
                .count                = WOL_MODE_COUNT,
                .strings        = wol_mode_names,
        },
        [ETH_SS_SOF_TIMESTAMPING] = {
                .per_dev        = false,
                .count                = __SOF_TIMESTAMPING_CNT,
                .strings        = sof_timestamping_names,
        },
        [ETH_SS_TS_TX_TYPES] = {
                .per_dev        = false,
                .count                = __HWTSTAMP_TX_CNT,
                .strings        = ts_tx_type_names,
        },
        [ETH_SS_TS_RX_FILTERS] = {
                .per_dev        = false,
                .count                = __HWTSTAMP_FILTER_CNT,
                .strings        = ts_rx_filter_names,
        },
        [ETH_SS_UDP_TUNNEL_TYPES] = {
                .per_dev        = false,
                .count                = __ETHTOOL_UDP_TUNNEL_TYPE_CNT,
                .strings        = udp_tunnel_type_names,
        },
        [ETH_SS_STATS_STD] = {
                .per_dev        = false,
                .count                = __ETHTOOL_STATS_CNT,
                .strings        = stats_std_names,
        },
        [ETH_SS_STATS_ETH_PHY] = {
                .per_dev        = false,
                .count                = __ETHTOOL_A_STATS_ETH_PHY_CNT,
                .strings        = stats_eth_phy_names,
        },
        [ETH_SS_STATS_ETH_MAC] = {
                .per_dev        = false,
                .count                = __ETHTOOL_A_STATS_ETH_MAC_CNT,
                .strings        = stats_eth_mac_names,
        },
        [ETH_SS_STATS_ETH_CTRL] = {
                .per_dev        = false,
                .count                = __ETHTOOL_A_STATS_ETH_CTRL_CNT,
                .strings        = stats_eth_ctrl_names,
        },
        [ETH_SS_STATS_RMON] = {
                .per_dev        = false,
                .count                = __ETHTOOL_A_STATS_RMON_CNT,
                .strings        = stats_rmon_names,
        },
};

struct strset_req_info {
        struct ethnl_req_info                base;
        u32                                req_ids;
        bool                                counts_only;
};

#define STRSET_REQINFO(__req_base) \
        container_of(__req_base, struct strset_req_info, base)

struct strset_reply_data {
        struct ethnl_reply_data                base;
        struct strset_info                sets[ETH_SS_COUNT];
};

#define STRSET_REPDATA(__reply_base) \
        container_of(__reply_base, struct strset_reply_data, base)

const struct nla_policy ethnl_strset_get_policy[] = {
        [ETHTOOL_A_STRSET_HEADER]        =
                NLA_POLICY_NESTED(ethnl_header_policy_phy),
        [ETHTOOL_A_STRSET_STRINGSETS]        = { .type = NLA_NESTED },
        [ETHTOOL_A_STRSET_COUNTS_ONLY]        = { .type = NLA_FLAG },
};

static const struct nla_policy get_stringset_policy[] = {
        [ETHTOOL_A_STRINGSET_ID]        = { .type = NLA_U32 },
};

/**
 * strset_include() - test if a string set should be included in reply
 * @info: parsed client request
 * @data: pointer to request data structure
 * @id:   id of string set to check (ETH_SS_* constants)
 */
static bool strset_include(const struct strset_req_info *info,
                           const struct strset_reply_data *data, u32 id)
{
        bool per_dev;

        BUILD_BUG_ON(ETH_SS_COUNT >= BITS_PER_BYTE * sizeof(info->req_ids));

        if (info->req_ids)
                return info->req_ids & (1U << id);
        per_dev = data->sets[id].per_dev;
        if (!per_dev && !data->sets[id].strings)
                return false;

        return data->base.dev ? per_dev : !per_dev;
}

static int strset_get_id(const struct nlattr *nest, u32 *val,
                         struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(get_stringset_policy)];
        int ret;

        ret = nla_parse_nested(tb, ARRAY_SIZE(get_stringset_policy) - 1, nest,
                               get_stringset_policy, extack);
        if (ret < 0)
                return ret;
        if (NL_REQ_ATTR_CHECK(extack, nest, tb, ETHTOOL_A_STRINGSET_ID))
                return -EINVAL;

        *val = nla_get_u32(tb[ETHTOOL_A_STRINGSET_ID]);
        return 0;
}

static const struct nla_policy strset_stringsets_policy[] = {
        [ETHTOOL_A_STRINGSETS_STRINGSET]        = { .type = NLA_NESTED },
};

static int strset_parse_request(struct ethnl_req_info *req_base,
                                struct nlattr **tb,
                                struct netlink_ext_ack *extack)
{
        struct strset_req_info *req_info = STRSET_REQINFO(req_base);
        struct nlattr *nest = tb[ETHTOOL_A_STRSET_STRINGSETS];
        struct nlattr *attr;
        int rem, ret;

        if (!nest)
                return 0;
        ret = nla_validate_nested(nest,
                                  ARRAY_SIZE(strset_stringsets_policy) - 1,
                                  strset_stringsets_policy, extack);
        if (ret < 0)
                return ret;

        req_info->counts_only = tb[ETHTOOL_A_STRSET_COUNTS_ONLY];
        nla_for_each_nested(attr, nest, rem) {
                u32 id;

                if (WARN_ONCE(nla_type(attr) != ETHTOOL_A_STRINGSETS_STRINGSET,
                              "unexpected attrtype %u in ETHTOOL_A_STRSET_STRINGSETS\n",
                              nla_type(attr)))
                        return -EINVAL;

                ret = strset_get_id(attr, &id, extack);
                if (ret < 0)
                        return ret;
                if (id >= ETH_SS_COUNT) {
                        NL_SET_ERR_MSG_ATTR(extack, attr,
                                            "unknown string set id");
                        return -EOPNOTSUPP;
                }

                req_info->req_ids |= (1U << id);
        }

        return 0;
}

static void strset_cleanup_data(struct ethnl_reply_data *reply_base)
{
        struct strset_reply_data *data = STRSET_REPDATA(reply_base);
        unsigned int i;

        for (i = 0; i < ETH_SS_COUNT; i++)
                if (data->sets[i].free_strings) {
                        kfree(data->sets[i].strings);
                        data->sets[i].strings = NULL;
                        data->sets[i].free_strings = false;
                }
}

static int strset_prepare_set(struct strset_info *info, struct net_device *dev,
                              struct phy_device *phydev, unsigned int id,
                              bool counts_only)
{
        const struct ethtool_phy_ops *phy_ops = ethtool_phy_ops;
        const struct ethtool_ops *ops = dev->ethtool_ops;
        void *strings;
        int count, ret;

        if (id == ETH_SS_PHY_STATS && phydev &&
            !ops->get_ethtool_phy_stats && phy_ops &&
            phy_ops->get_sset_count)
                ret = phy_ops->get_sset_count(phydev);
        else if (ops->get_sset_count && ops->get_strings)
                ret = ops->get_sset_count(dev, id);
        else
                ret = -EOPNOTSUPP;
        if (ret <= 0) {
                info->count = 0;
                return 0;
        }

        count = ret;
        if (!counts_only) {
                strings = kcalloc(count, ETH_GSTRING_LEN, GFP_KERNEL);
                if (!strings)
                        return -ENOMEM;
                if (id == ETH_SS_PHY_STATS && phydev &&
                    !ops->get_ethtool_phy_stats && phy_ops &&
                    phy_ops->get_strings)
                        phy_ops->get_strings(phydev, strings);
                else
                        ops->get_strings(dev, id, strings);
                info->strings = strings;
                info->free_strings = true;
        }
        info->count = count;

        return 0;
}

static int strset_prepare_data(const struct ethnl_req_info *req_base,
                               struct ethnl_reply_data *reply_base,
                               const struct genl_info *info)
{
        const struct strset_req_info *req_info = STRSET_REQINFO(req_base);
        struct strset_reply_data *data = STRSET_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        struct nlattr **tb = info->attrs;
        struct phy_device *phydev;
        unsigned int i;
        int ret;

        BUILD_BUG_ON(ARRAY_SIZE(info_template) != ETH_SS_COUNT);
        memcpy(&data->sets, &info_template, sizeof(data->sets));

        if (!dev) {
                for (i = 0; i < ETH_SS_COUNT; i++) {
                        if ((req_info->req_ids & (1U << i)) &&
                            data->sets[i].per_dev) {
                                GENL_SET_ERR_MSG(info, "requested per device strings without dev");
                                return -EINVAL;
                        }
                }
                return 0;
        }

        phydev = ethnl_req_get_phydev(req_base, tb[ETHTOOL_A_HEADER_FLAGS],
                                      info->extack);

        /* phydev can be NULL, check for errors only */
        if (IS_ERR(phydev))
                return PTR_ERR(phydev);

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                goto err_strset;
        for (i = 0; i < ETH_SS_COUNT; i++) {
                if (!strset_include(req_info, data, i) ||
                    !data->sets[i].per_dev)
                        continue;

                ret = strset_prepare_set(&data->sets[i], dev, phydev, i,
                                         req_info->counts_only);
                if (ret < 0)
                        goto err_ops;
        }
        ethnl_ops_complete(dev);

        return 0;
err_ops:
        ethnl_ops_complete(dev);
err_strset:
        strset_cleanup_data(reply_base);
        return ret;
}

/* calculate size of ETHTOOL_A_STRSET_STRINGSET nest for one string set */
static int strset_set_size(const struct strset_info *info, bool counts_only)
{
        unsigned int len = 0;
        unsigned int i;

        if (info->count == 0)
                return 0;
        if (counts_only)
                return nla_total_size(2 * nla_total_size(sizeof(u32)));

        for (i = 0; i < info->count; i++) {
                const char *str = info->strings[i];

                /* ETHTOOL_A_STRING_INDEX, ETHTOOL_A_STRING_VALUE, nest */
                len += nla_total_size(nla_total_size(sizeof(u32)) +
                                      ethnl_strz_size(str));
        }
        /* ETHTOOL_A_STRINGSET_ID, ETHTOOL_A_STRINGSET_COUNT */
        len = 2 * nla_total_size(sizeof(u32)) + nla_total_size(len);

        return nla_total_size(len);
}

static int strset_reply_size(const struct ethnl_req_info *req_base,
                             const struct ethnl_reply_data *reply_base)
{
        const struct strset_req_info *req_info = STRSET_REQINFO(req_base);
        const struct strset_reply_data *data = STRSET_REPDATA(reply_base);
        unsigned int i;
        int len = 0;
        int ret;

        len += nla_total_size(0); /* ETHTOOL_A_STRSET_STRINGSETS */

        for (i = 0; i < ETH_SS_COUNT; i++) {
                const struct strset_info *set_info = &data->sets[i];

                if (!strset_include(req_info, data, i))
                        continue;

                ret = strset_set_size(set_info, req_info->counts_only);
                if (ret < 0)
                        return ret;
                len += ret;
        }

        return len;
}

/* fill one string into reply */
static int strset_fill_string(struct sk_buff *skb,
                              const struct strset_info *set_info, u32 idx)
{
        struct nlattr *string_attr;
        const char *value;

        value = set_info->strings[idx];

        string_attr = nla_nest_start(skb, ETHTOOL_A_STRINGS_STRING);
        if (!string_attr)
                return -EMSGSIZE;
        if (nla_put_u32(skb, ETHTOOL_A_STRING_INDEX, idx) ||
            ethnl_put_strz(skb, ETHTOOL_A_STRING_VALUE, value))
                goto nla_put_failure;
        nla_nest_end(skb, string_attr);

        return 0;
nla_put_failure:
        nla_nest_cancel(skb, string_attr);
        return -EMSGSIZE;
}

/* fill one string set into reply */
static int strset_fill_set(struct sk_buff *skb,
                           const struct strset_info *set_info, u32 id,
                           bool counts_only)
{
        struct nlattr *stringset_attr;
        struct nlattr *strings_attr;
        unsigned int i;

        if (!set_info->per_dev && !set_info->strings)
                return -EOPNOTSUPP;
        if (set_info->count == 0)
                return 0;
        stringset_attr = nla_nest_start(skb, ETHTOOL_A_STRINGSETS_STRINGSET);
        if (!stringset_attr)
                return -EMSGSIZE;

        if (nla_put_u32(skb, ETHTOOL_A_STRINGSET_ID, id) ||
            nla_put_u32(skb, ETHTOOL_A_STRINGSET_COUNT, set_info->count))
                goto nla_put_failure;

        if (!counts_only) {
                strings_attr = nla_nest_start(skb, ETHTOOL_A_STRINGSET_STRINGS);
                if (!strings_attr)
                        goto nla_put_failure;
                for (i = 0; i < set_info->count; i++) {
                        if (strset_fill_string(skb, set_info, i) < 0)
                                goto nla_put_failure;
                }
                nla_nest_end(skb, strings_attr);
        }

        nla_nest_end(skb, stringset_attr);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, stringset_attr);
        return -EMSGSIZE;
}

static int strset_fill_reply(struct sk_buff *skb,
                             const struct ethnl_req_info *req_base,
                             const struct ethnl_reply_data *reply_base)
{
        const struct strset_req_info *req_info = STRSET_REQINFO(req_base);
        const struct strset_reply_data *data = STRSET_REPDATA(reply_base);
        struct nlattr *nest;
        unsigned int i;
        int ret;

        nest = nla_nest_start(skb, ETHTOOL_A_STRSET_STRINGSETS);
        if (!nest)
                return -EMSGSIZE;

        for (i = 0; i < ETH_SS_COUNT; i++) {
                if (strset_include(req_info, data, i)) {
                        ret = strset_fill_set(skb, &data->sets[i], i,
                                              req_info->counts_only);
                        if (ret < 0)
                                goto nla_put_failure;
                }
        }

        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return ret;
}

const struct ethnl_request_ops ethnl_strset_request_ops = {
        .request_cmd                = ETHTOOL_MSG_STRSET_GET,
        .reply_cmd                = ETHTOOL_MSG_STRSET_GET_REPLY,
        .hdr_attr                = ETHTOOL_A_STRSET_HEADER,
        .req_info_size                = sizeof(struct strset_req_info),
        .reply_data_size        = sizeof(struct strset_reply_data),
        .allow_nodev_do                = true,

        .parse_request                = strset_parse_request,
        .prepare_data                = strset_prepare_data,
        .reply_size                = strset_reply_size,
        .fill_reply                = strset_fill_reply,
        .cleanup_data                = strset_cleanup_data,
};































































































































































  111 


  111 
















  237 
  299 

















































    3 
    2 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_GENERIC_SECTIONS_H_
#define _ASM_GENERIC_SECTIONS_H_

/* References to section boundaries */

#include <linux/compiler.h>
#include <linux/types.h>

/*
 * Usage guidelines:
 * _text, _data: architecture specific, don't use them in arch-independent code
 * [_stext, _etext]: contains .text.* sections, may also contain .rodata.*
 *                   and/or .init.* sections
 * [_sdata, _edata]: contains .data.* sections, may also contain .rodata.*
 *                   and/or .init.* sections.
 * [__start_rodata, __end_rodata]: contains .rodata.* sections
 * [__start_ro_after_init, __end_ro_after_init]:
 *                     contains .data..ro_after_init section
 * [__init_begin, __init_end]: contains .init.* sections, but .init.text.*
 *                   may be out of this range on some architectures.
 * [_sinittext, _einittext]: contains .init.text.* sections
 * [__bss_start, __bss_stop]: contains BSS sections
 *
 * Following global variables are optional and may be unavailable on some
 * architectures and/or kernel configurations.
 *        _text, _data
 *        __kprobes_text_start, __kprobes_text_end
 *        __entry_text_start, __entry_text_end
 *        __ctors_start, __ctors_end
 *        __irqentry_text_start, __irqentry_text_end
 *        __softirqentry_text_start, __softirqentry_text_end
 *        __start_opd, __end_opd
 */
extern char _text[], _stext[], _etext[];
extern char _data[], _sdata[], _edata[];
extern char __bss_start[], __bss_stop[];
extern char __init_begin[], __init_end[];
extern char _sinittext[], _einittext[];
extern char __start_ro_after_init[], __end_ro_after_init[];
extern char _end[];
extern char __per_cpu_load[], __per_cpu_start[], __per_cpu_end[];
extern char __kprobes_text_start[], __kprobes_text_end[];
extern char __entry_text_start[], __entry_text_end[];
extern char __start_rodata[], __end_rodata[];
extern char __irqentry_text_start[], __irqentry_text_end[];
extern char __softirqentry_text_start[], __softirqentry_text_end[];
extern char __start_once[], __end_once[];

/* Start and end of .ctors section - used for constructor calls. */
extern char __ctors_start[], __ctors_end[];

/* Start and end of .opd section - used for function descriptors. */
extern char __start_opd[], __end_opd[];

/* Start and end of instrumentation protected text section */
extern char __noinstr_text_start[], __noinstr_text_end[];

extern __visible const void __nosave_begin, __nosave_end;

/* Function descriptor handling (if any).  Override in asm/sections.h */
#ifdef CONFIG_HAVE_FUNCTION_DESCRIPTORS
void *dereference_function_descriptor(void *ptr);
void *dereference_kernel_function_descriptor(void *ptr);
#else
#define dereference_function_descriptor(p) ((void *)(p))
#define dereference_kernel_function_descriptor(p) ((void *)(p))

/* An address is simply the address of the function. */
typedef struct {
        unsigned long addr;
} func_desc_t;
#endif

static inline bool have_function_descriptors(void)
{
        return IS_ENABLED(CONFIG_HAVE_FUNCTION_DESCRIPTORS);
}

/**
 * memory_contains - checks if an object is contained within a memory region
 * @begin: virtual address of the beginning of the memory region
 * @end: virtual address of the end of the memory region
 * @virt: virtual address of the memory object
 * @size: size of the memory object
 *
 * Returns: true if the object specified by @virt and @size is entirely
 * contained within the memory region defined by @begin and @end, false
 * otherwise.
 */
static inline bool memory_contains(void *begin, void *end, void *virt,
                                   size_t size)
{
        return virt >= begin && virt + size <= end;
}

/**
 * memory_intersects - checks if the region occupied by an object intersects
 *                     with another memory region
 * @begin: virtual address of the beginning of the memory region
 * @end: virtual address of the end of the memory region
 * @virt: virtual address of the memory object
 * @size: size of the memory object
 *
 * Returns: true if an object's memory region, specified by @virt and @size,
 * intersects with the region specified by @begin and @end, false otherwise.
 */
static inline bool memory_intersects(void *begin, void *end, void *virt,
                                     size_t size)
{
        void *vend = virt + size;

        if (virt < end && vend > begin)
                return true;

        return false;
}

/**
 * init_section_contains - checks if an object is contained within the init
 *                         section
 * @virt: virtual address of the memory object
 * @size: size of the memory object
 *
 * Returns: true if the object specified by @virt and @size is entirely
 * contained within the init section, false otherwise.
 */
static inline bool init_section_contains(void *virt, size_t size)
{
        return memory_contains(__init_begin, __init_end, virt, size);
}

/**
 * init_section_intersects - checks if the region occupied by an object
 *                           intersects with the init section
 * @virt: virtual address of the memory object
 * @size: size of the memory object
 *
 * Returns: true if an object's memory region, specified by @virt and @size,
 * intersects with the init section, false otherwise.
 */
static inline bool init_section_intersects(void *virt, size_t size)
{
        return memory_intersects(__init_begin, __init_end, virt, size);
}

/**
 * is_kernel_core_data - checks if the pointer address is located in the
 *                         .data or .bss section
 *
 * @addr: address to check
 *
 * Returns: true if the address is located in .data or .bss, false otherwise.
 * Note: On some archs it may return true for core RODATA, and false
 *       for others. But will always be true for core RW data.
 */
static inline bool is_kernel_core_data(unsigned long addr)
{
        if (addr >= (unsigned long)_sdata && addr < (unsigned long)_edata)
                return true;

        if (addr >= (unsigned long)__bss_start &&
            addr < (unsigned long)__bss_stop)
                return true;

        return false;
}

/**
 * is_kernel_rodata - checks if the pointer address is located in the
 *                    .rodata section
 *
 * @addr: address to check
 *
 * Returns: true if the address is located in .rodata, false otherwise.
 */
static inline bool is_kernel_rodata(unsigned long addr)
{
        return addr >= (unsigned long)__start_rodata &&
               addr < (unsigned long)__end_rodata;
}

static inline bool is_kernel_ro_after_init(unsigned long addr)
{
        return addr >= (unsigned long)__start_ro_after_init &&
               addr < (unsigned long)__end_ro_after_init;
}
/**
 * is_kernel_inittext - checks if the pointer address is located in the
 *                      .init.text section
 *
 * @addr: address to check
 *
 * Returns: true if the address is located in .init.text, false otherwise.
 */
static inline bool is_kernel_inittext(unsigned long addr)
{
        return addr >= (unsigned long)_sinittext &&
               addr < (unsigned long)_einittext;
}

/**
 * __is_kernel_text - checks if the pointer address is located in the
 *                    .text section
 *
 * @addr: address to check
 *
 * Returns: true if the address is located in .text, false otherwise.
 * Note: an internal helper, only check the range of _stext to _etext.
 */
static inline bool __is_kernel_text(unsigned long addr)
{
        return addr >= (unsigned long)_stext &&
               addr < (unsigned long)_etext;
}

/**
 * __is_kernel - checks if the pointer address is located in the kernel range
 *
 * @addr: address to check
 *
 * Returns: true if the address is located in the kernel range, false otherwise.
 * Note: an internal helper, check the range of _stext to _end,
 *       and range from __init_begin to __init_end, which can be outside
 *       of the _stext to _end range.
 */
static inline bool __is_kernel(unsigned long addr)
{
        return ((addr >= (unsigned long)_stext &&
                 addr < (unsigned long)_end) ||
                (addr >= (unsigned long)__init_begin &&
                 addr < (unsigned long)__init_end));
}

#endif /* _ASM_GENERIC_SECTIONS_H_ */




















   43 











   26 










  185 












  185 











   18 

    2 
   20 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *        Skb ref helpers.
 *
 */

#ifndef _LINUX_SKBUFF_REF_H
#define _LINUX_SKBUFF_REF_H

#include <linux/skbuff.h>

/**
 * __skb_frag_ref - take an addition reference on a paged fragment.
 * @frag: the paged fragment
 *
 * Takes an additional reference on the paged fragment @frag.
 */
static inline void __skb_frag_ref(skb_frag_t *frag)
{
        get_page(skb_frag_page(frag));
}

/**
 * skb_frag_ref - take an addition reference on a paged fragment of an skb.
 * @skb: the buffer
 * @f: the fragment offset.
 *
 * Takes an additional reference on the @f'th paged fragment of @skb.
 */
static inline void skb_frag_ref(struct sk_buff *skb, int f)
{
        __skb_frag_ref(&skb_shinfo(skb)->frags[f]);
}

bool napi_pp_put_page(netmem_ref netmem);

static inline void skb_page_unref(netmem_ref netmem, bool recycle)
{
#ifdef CONFIG_PAGE_POOL
        if (recycle && napi_pp_put_page(netmem))
                return;
#endif
        put_page(netmem_to_page(netmem));
}

/**
 * __skb_frag_unref - release a reference on a paged fragment.
 * @frag: the paged fragment
 * @recycle: recycle the page if allocated via page_pool
 *
 * Releases a reference on the paged fragment @frag
 * or recycles the page via the page_pool API.
 */
static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle)
{
        skb_page_unref(skb_frag_netmem(frag), recycle);
}

/**
 * skb_frag_unref - release a reference on a paged fragment of an skb.
 * @skb: the buffer
 * @f: the fragment offset
 *
 * Releases a reference on the @f'th paged fragment of @skb.
 */
static inline void skb_frag_unref(struct sk_buff *skb, int f)
{
        struct skb_shared_info *shinfo = skb_shinfo(skb);

        if (!skb_zcopy_managed(skb))
                __skb_frag_unref(&shinfo->frags[f], skb->pp_recycle);
}

#endif        /* _LINUX_SKBUFF_REF_H */

































































































































































































































































    5 


    4 




    6 








    4 

    3 

































   12 


   13 


   12 





   11 







































   48 




















































































  157 












  221 




  221 








   13 








   13 












   13 



   13 




   21 
















   21 




























   21 

   21 
   25 
   16 
   13 




















    9 











   41 

   41 








































   31 



















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_IP_TUNNELS_H
#define __NET_IP_TUNNELS_H 1

#include <linux/if_tunnel.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/socket.h>
#include <linux/types.h>
#include <linux/u64_stats_sync.h>
#include <linux/bitops.h>

#include <net/dsfield.h>
#include <net/gro_cells.h>
#include <net/inet_ecn.h>
#include <net/netns/generic.h>
#include <net/rtnetlink.h>
#include <net/lwtunnel.h>
#include <net/dst_cache.h>

#if IS_ENABLED(CONFIG_IPV6)
#include <net/ipv6.h>
#include <net/ip6_fib.h>
#include <net/ip6_route.h>
#endif

/* Keep error state on tunnel for 30 sec */
#define IPTUNNEL_ERR_TIMEO        (30*HZ)

/* Used to memset ip_tunnel padding. */
#define IP_TUNNEL_KEY_SIZE        offsetofend(struct ip_tunnel_key, tp_dst)

/* Used to memset ipv4 address padding. */
#define IP_TUNNEL_KEY_IPV4_PAD        offsetofend(struct ip_tunnel_key, u.ipv4.dst)
#define IP_TUNNEL_KEY_IPV4_PAD_LEN                                \
        (sizeof_field(struct ip_tunnel_key, u) -                \
         sizeof_field(struct ip_tunnel_key, u.ipv4))

#define __ipt_flag_op(op, ...)                                        \
        op(__VA_ARGS__, __IP_TUNNEL_FLAG_NUM)

#define IP_TUNNEL_DECLARE_FLAGS(...)                                \
        __ipt_flag_op(DECLARE_BITMAP, __VA_ARGS__)

#define ip_tunnel_flags_zero(...)        __ipt_flag_op(bitmap_zero, __VA_ARGS__)
#define ip_tunnel_flags_copy(...)        __ipt_flag_op(bitmap_copy, __VA_ARGS__)
#define ip_tunnel_flags_and(...)        __ipt_flag_op(bitmap_and, __VA_ARGS__)
#define ip_tunnel_flags_or(...)                __ipt_flag_op(bitmap_or, __VA_ARGS__)

#define ip_tunnel_flags_empty(...)                                \
        __ipt_flag_op(bitmap_empty, __VA_ARGS__)
#define ip_tunnel_flags_intersect(...)                                \
        __ipt_flag_op(bitmap_intersects, __VA_ARGS__)
#define ip_tunnel_flags_subset(...)                                \
        __ipt_flag_op(bitmap_subset, __VA_ARGS__)

struct ip_tunnel_key {
        __be64                        tun_id;
        union {
                struct {
                        __be32        src;
                        __be32        dst;
                } ipv4;
                struct {
                        struct in6_addr src;
                        struct in6_addr dst;
                } ipv6;
        } u;
        IP_TUNNEL_DECLARE_FLAGS(tun_flags);
        __be32                        label;                /* Flow Label for IPv6 */
        u32                        nhid;
        u8                        tos;                /* TOS for IPv4, TC for IPv6 */
        u8                        ttl;                /* TTL for IPv4, HL for IPv6 */
        __be16                        tp_src;
        __be16                        tp_dst;
        __u8                        flow_flags;
};

struct ip_tunnel_encap {
        u16                        type;
        u16                        flags;
        __be16                        sport;
        __be16                        dport;
};

/* Flags for ip_tunnel_info mode. */
#define IP_TUNNEL_INFO_TX        0x01        /* represents tx tunnel parameters */
#define IP_TUNNEL_INFO_IPV6        0x02        /* key contains IPv6 addresses */
#define IP_TUNNEL_INFO_BRIDGE        0x04        /* represents a bridged tunnel id */

/* Maximum tunnel options length. */
#define IP_TUNNEL_OPTS_MAX                                        \
        GENMASK((sizeof_field(struct ip_tunnel_info,                \
                              options_len) * BITS_PER_BYTE) - 1, 0)

#define ip_tunnel_info_opts(info)                                \
        _Generic(info,                                                \
                 const struct ip_tunnel_info * : ((const void *)((info) + 1)),\
                 struct ip_tunnel_info * : ((void *)((info) + 1))\
        )

struct ip_tunnel_info {
        struct ip_tunnel_key        key;
        struct ip_tunnel_encap        encap;
#ifdef CONFIG_DST_CACHE
        struct dst_cache        dst_cache;
#endif
        u8                        options_len;
        u8                        mode;
};

/* 6rd prefix/relay information */
#ifdef CONFIG_IPV6_SIT_6RD
struct ip_tunnel_6rd_parm {
        struct in6_addr                prefix;
        __be32                        relay_prefix;
        u16                        prefixlen;
        u16                        relay_prefixlen;
};
#endif

struct ip_tunnel_prl_entry {
        struct ip_tunnel_prl_entry __rcu *next;
        __be32                                addr;
        u16                                flags;
        struct rcu_head                        rcu_head;
};

struct metadata_dst;

/* Kernel-side variant of ip_tunnel_parm */
struct ip_tunnel_parm_kern {
        char                        name[IFNAMSIZ];
        IP_TUNNEL_DECLARE_FLAGS(i_flags);
        IP_TUNNEL_DECLARE_FLAGS(o_flags);
        __be32                        i_key;
        __be32                        o_key;
        int                        link;
        struct iphdr                iph;
};

struct ip_tunnel {
        struct ip_tunnel __rcu        *next;
        struct hlist_node hash_node;

        struct net_device        *dev;
        netdevice_tracker        dev_tracker;

        struct net                *net;        /* netns for packet i/o */

        unsigned long        err_time;        /* Time when the last ICMP error
                                         * arrived */
        int                err_count;        /* Number of arrived ICMP errors */

        /* These four fields used only by GRE */
        u32                i_seqno;        /* The last seen seqno        */
        atomic_t        o_seqno;        /* The last output seqno */
        int                tun_hlen;        /* Precalculated header length */

        /* These four fields used only by ERSPAN */
        u32                index;                /* ERSPAN type II index */
        u8                erspan_ver;        /* ERSPAN version */
        u8                dir;                /* ERSPAN direction */
        u16                hwid;                /* ERSPAN hardware ID */

        struct dst_cache dst_cache;

        struct ip_tunnel_parm_kern parms;

        int                mlink;
        int                encap_hlen;        /* Encap header length (FOU,GUE) */
        int                hlen;                /* tun_hlen + encap_hlen */
        struct ip_tunnel_encap encap;

        /* for SIT */
#ifdef CONFIG_IPV6_SIT_6RD
        struct ip_tunnel_6rd_parm ip6rd;
#endif
        struct ip_tunnel_prl_entry __rcu *prl;        /* potential router list */
        unsigned int                prl_count;        /* # of entries in PRL */
        unsigned int                ip_tnl_net_id;
        struct gro_cells        gro_cells;
        __u32                        fwmark;
        bool                        collect_md;
        bool                        ignore_df;
};

struct tnl_ptk_info {
        IP_TUNNEL_DECLARE_FLAGS(flags);
        __be16 proto;
        __be32 key;
        __be32 seq;
        int hdr_len;
};

#define PACKET_RCVD        0
#define PACKET_REJECT        1
#define PACKET_NEXT        2

#define IP_TNL_HASH_BITS   7
#define IP_TNL_HASH_SIZE   (1 << IP_TNL_HASH_BITS)

struct ip_tunnel_net {
        struct net_device *fb_tunnel_dev;
        struct rtnl_link_ops *rtnl_link_ops;
        struct hlist_head tunnels[IP_TNL_HASH_SIZE];
        struct ip_tunnel __rcu *collect_md_tun;
        int type;
};

static inline void ip_tunnel_set_options_present(unsigned long *flags)
{
        IP_TUNNEL_DECLARE_FLAGS(present) = { };

        __set_bit(IP_TUNNEL_GENEVE_OPT_BIT, present);
        __set_bit(IP_TUNNEL_VXLAN_OPT_BIT, present);
        __set_bit(IP_TUNNEL_ERSPAN_OPT_BIT, present);
        __set_bit(IP_TUNNEL_GTP_OPT_BIT, present);
        __set_bit(IP_TUNNEL_PFCP_OPT_BIT, present);

        ip_tunnel_flags_or(flags, flags, present);
}

static inline void ip_tunnel_clear_options_present(unsigned long *flags)
{
        IP_TUNNEL_DECLARE_FLAGS(present) = { };

        __set_bit(IP_TUNNEL_GENEVE_OPT_BIT, present);
        __set_bit(IP_TUNNEL_VXLAN_OPT_BIT, present);
        __set_bit(IP_TUNNEL_ERSPAN_OPT_BIT, present);
        __set_bit(IP_TUNNEL_GTP_OPT_BIT, present);
        __set_bit(IP_TUNNEL_PFCP_OPT_BIT, present);

        __ipt_flag_op(bitmap_andnot, flags, flags, present);
}

static inline bool ip_tunnel_is_options_present(const unsigned long *flags)
{
        IP_TUNNEL_DECLARE_FLAGS(present) = { };

        __set_bit(IP_TUNNEL_GENEVE_OPT_BIT, present);
        __set_bit(IP_TUNNEL_VXLAN_OPT_BIT, present);
        __set_bit(IP_TUNNEL_ERSPAN_OPT_BIT, present);
        __set_bit(IP_TUNNEL_GTP_OPT_BIT, present);
        __set_bit(IP_TUNNEL_PFCP_OPT_BIT, present);

        return ip_tunnel_flags_intersect(flags, present);
}

static inline bool ip_tunnel_flags_is_be16_compat(const unsigned long *flags)
{
        IP_TUNNEL_DECLARE_FLAGS(supp) = { };

        bitmap_set(supp, 0, BITS_PER_TYPE(__be16));
        __set_bit(IP_TUNNEL_VTI_BIT, supp);

        return ip_tunnel_flags_subset(flags, supp);
}

static inline void ip_tunnel_flags_from_be16(unsigned long *dst, __be16 flags)
{
        ip_tunnel_flags_zero(dst);

        bitmap_write(dst, be16_to_cpu(flags), 0, BITS_PER_TYPE(__be16));
        __assign_bit(IP_TUNNEL_VTI_BIT, dst, flags & VTI_ISVTI);
}

static inline __be16 ip_tunnel_flags_to_be16(const unsigned long *flags)
{
        __be16 ret;

        ret = cpu_to_be16(bitmap_read(flags, 0, BITS_PER_TYPE(__be16)));
        if (test_bit(IP_TUNNEL_VTI_BIT, flags))
                ret |= VTI_ISVTI;

        return ret;
}

static inline void ip_tunnel_key_init(struct ip_tunnel_key *key,
                                      __be32 saddr, __be32 daddr,
                                      u8 tos, u8 ttl, __be32 label,
                                      __be16 tp_src, __be16 tp_dst,
                                      __be64 tun_id,
                                      const unsigned long *tun_flags)
{
        key->tun_id = tun_id;
        key->u.ipv4.src = saddr;
        key->u.ipv4.dst = daddr;
        memset((unsigned char *)key + IP_TUNNEL_KEY_IPV4_PAD,
               0, IP_TUNNEL_KEY_IPV4_PAD_LEN);
        key->tos = tos;
        key->ttl = ttl;
        key->label = label;
        ip_tunnel_flags_copy(key->tun_flags, tun_flags);

        /* For the tunnel types on the top of IPsec, the tp_src and tp_dst of
         * the upper tunnel are used.
         * E.g: GRE over IPSEC, the tp_src and tp_port are zero.
         */
        key->tp_src = tp_src;
        key->tp_dst = tp_dst;

        /* Clear struct padding. */
        if (sizeof(*key) != IP_TUNNEL_KEY_SIZE)
                memset((unsigned char *)key + IP_TUNNEL_KEY_SIZE,
                       0, sizeof(*key) - IP_TUNNEL_KEY_SIZE);
}

static inline bool
ip_tunnel_dst_cache_usable(const struct sk_buff *skb,
                           const struct ip_tunnel_info *info)
{
        if (skb->mark)
                return false;

        return !info || !test_bit(IP_TUNNEL_NOCACHE_BIT, info->key.tun_flags);
}

static inline unsigned short ip_tunnel_info_af(const struct ip_tunnel_info
                                               *tun_info)
{
        return tun_info->mode & IP_TUNNEL_INFO_IPV6 ? AF_INET6 : AF_INET;
}

static inline __be64 key32_to_tunnel_id(__be32 key)
{
#ifdef __BIG_ENDIAN
        return (__force __be64)key;
#else
        return (__force __be64)((__force u64)key << 32);
#endif
}

/* Returns the least-significant 32 bits of a __be64. */
static inline __be32 tunnel_id_to_key32(__be64 tun_id)
{
#ifdef __BIG_ENDIAN
        return (__force __be32)tun_id;
#else
        return (__force __be32)((__force u64)tun_id >> 32);
#endif
}

#ifdef CONFIG_INET

static inline void ip_tunnel_init_flow(struct flowi4 *fl4,
                                       int proto,
                                       __be32 daddr, __be32 saddr,
                                       __be32 key, __u8 tos,
                                       struct net *net, int oif,
                                       __u32 mark, __u32 tun_inner_hash,
                                       __u8 flow_flags)
{
        memset(fl4, 0, sizeof(*fl4));

        if (oif) {
                fl4->flowi4_l3mdev = l3mdev_master_upper_ifindex_by_index(net, oif);
                /* Legacy VRF/l3mdev use case */
                fl4->flowi4_oif = fl4->flowi4_l3mdev ? 0 : oif;
        }

        fl4->daddr = daddr;
        fl4->saddr = saddr;
        fl4->flowi4_tos = tos;
        fl4->flowi4_proto = proto;
        fl4->fl4_gre_key = key;
        fl4->flowi4_mark = mark;
        fl4->flowi4_multipath_hash = tun_inner_hash;
        fl4->flowi4_flags = flow_flags;
}

int ip_tunnel_init(struct net_device *dev);
void ip_tunnel_uninit(struct net_device *dev);
void  ip_tunnel_dellink(struct net_device *dev, struct list_head *head);
struct net *ip_tunnel_get_link_net(const struct net_device *dev);
int ip_tunnel_get_iflink(const struct net_device *dev);
int ip_tunnel_init_net(struct net *net, unsigned int ip_tnl_net_id,
                       struct rtnl_link_ops *ops, char *devname);

void ip_tunnel_delete_nets(struct list_head *list_net, unsigned int id,
                           struct rtnl_link_ops *ops,
                           struct list_head *dev_to_kill);

void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
                    const struct iphdr *tnl_params, const u8 protocol);
void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
                       const u8 proto, int tunnel_hlen);
int ip_tunnel_ctl(struct net_device *dev, struct ip_tunnel_parm_kern *p,
                  int cmd);
bool ip_tunnel_parm_from_user(struct ip_tunnel_parm_kern *kp,
                              const void __user *data);
bool ip_tunnel_parm_to_user(void __user *data, struct ip_tunnel_parm_kern *kp);
int ip_tunnel_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
                             void __user *data, int cmd);
int __ip_tunnel_change_mtu(struct net_device *dev, int new_mtu, bool strict);
int ip_tunnel_change_mtu(struct net_device *dev, int new_mtu);

struct ip_tunnel *ip_tunnel_lookup(struct ip_tunnel_net *itn,
                                   int link, const unsigned long *flags,
                                   __be32 remote, __be32 local,
                                   __be32 key);

void ip_tunnel_md_udp_encap(struct sk_buff *skb, struct ip_tunnel_info *info);
int ip_tunnel_rcv(struct ip_tunnel *tunnel, struct sk_buff *skb,
                  const struct tnl_ptk_info *tpi, struct metadata_dst *tun_dst,
                  bool log_ecn_error);
int ip_tunnel_changelink(struct net_device *dev, struct nlattr *tb[],
                         struct ip_tunnel_parm_kern *p, __u32 fwmark);
int ip_tunnel_newlink(struct net_device *dev, struct nlattr *tb[],
                      struct ip_tunnel_parm_kern *p, __u32 fwmark);
void ip_tunnel_setup(struct net_device *dev, unsigned int net_id);

bool ip_tunnel_netlink_encap_parms(struct nlattr *data[],
                                   struct ip_tunnel_encap *encap);

void ip_tunnel_netlink_parms(struct nlattr *data[],
                             struct ip_tunnel_parm_kern *parms);

extern const struct header_ops ip_tunnel_header_ops;
__be16 ip_tunnel_parse_protocol(const struct sk_buff *skb);

struct ip_tunnel_encap_ops {
        size_t (*encap_hlen)(struct ip_tunnel_encap *e);
        int (*build_header)(struct sk_buff *skb, struct ip_tunnel_encap *e,
                            u8 *protocol, struct flowi4 *fl4);
        int (*err_handler)(struct sk_buff *skb, u32 info);
};

#define MAX_IPTUN_ENCAP_OPS 8

extern const struct ip_tunnel_encap_ops __rcu *
                iptun_encaps[MAX_IPTUN_ENCAP_OPS];

int ip_tunnel_encap_add_ops(const struct ip_tunnel_encap_ops *op,
                            unsigned int num);
int ip_tunnel_encap_del_ops(const struct ip_tunnel_encap_ops *op,
                            unsigned int num);

int ip_tunnel_encap_setup(struct ip_tunnel *t,
                          struct ip_tunnel_encap *ipencap);

static inline enum skb_drop_reason
pskb_inet_may_pull_reason(struct sk_buff *skb)
{
        int nhlen;

        switch (skb->protocol) {
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                nhlen = sizeof(struct ipv6hdr);
                break;
#endif
        case htons(ETH_P_IP):
                nhlen = sizeof(struct iphdr);
                break;
        default:
                nhlen = 0;
        }

        return pskb_network_may_pull_reason(skb, nhlen);
}

static inline bool pskb_inet_may_pull(struct sk_buff *skb)
{
        return pskb_inet_may_pull_reason(skb) == SKB_NOT_DROPPED_YET;
}

/* Variant of pskb_inet_may_pull().
 */
static inline enum skb_drop_reason
skb_vlan_inet_prepare(struct sk_buff *skb, bool inner_proto_inherit)
{
        int nhlen = 0, maclen = inner_proto_inherit ? 0 : ETH_HLEN;
        __be16 type = skb->protocol;
        enum skb_drop_reason reason;

        /* Essentially this is skb_protocol(skb, true)
         * And we get MAC len.
         */
        if (eth_type_vlan(type))
                type = __vlan_get_protocol(skb, type, &maclen);

        switch (type) {
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                nhlen = sizeof(struct ipv6hdr);
                break;
#endif
        case htons(ETH_P_IP):
                nhlen = sizeof(struct iphdr);
                break;
        }
        /* For ETH_P_IPV6/ETH_P_IP we make sure to pull
         * a base network header in skb->head.
         */
        reason = pskb_may_pull_reason(skb, maclen + nhlen);
        if (reason)
                return reason;

        skb_set_network_header(skb, maclen);

        return SKB_NOT_DROPPED_YET;
}

static inline int ip_encap_hlen(struct ip_tunnel_encap *e)
{
        const struct ip_tunnel_encap_ops *ops;
        int hlen = -EINVAL;

        if (e->type == TUNNEL_ENCAP_NONE)
                return 0;

        if (e->type >= MAX_IPTUN_ENCAP_OPS)
                return -EINVAL;

        rcu_read_lock();
        ops = rcu_dereference(iptun_encaps[e->type]);
        if (likely(ops && ops->encap_hlen))
                hlen = ops->encap_hlen(e);
        rcu_read_unlock();

        return hlen;
}

static inline int ip_tunnel_encap(struct sk_buff *skb,
                                  struct ip_tunnel_encap *e,
                                  u8 *protocol, struct flowi4 *fl4)
{
        const struct ip_tunnel_encap_ops *ops;
        int ret = -EINVAL;

        if (e->type == TUNNEL_ENCAP_NONE)
                return 0;

        if (e->type >= MAX_IPTUN_ENCAP_OPS)
                return -EINVAL;

        rcu_read_lock();
        ops = rcu_dereference(iptun_encaps[e->type]);
        if (likely(ops && ops->build_header))
                ret = ops->build_header(skb, e, protocol, fl4);
        rcu_read_unlock();

        return ret;
}

/* Extract dsfield from inner protocol */
static inline u8 ip_tunnel_get_dsfield(const struct iphdr *iph,
                                       const struct sk_buff *skb)
{
        __be16 payload_protocol = skb_protocol(skb, true);

        if (payload_protocol == htons(ETH_P_IP))
                return iph->tos;
        else if (payload_protocol == htons(ETH_P_IPV6))
                return ipv6_get_dsfield((const struct ipv6hdr *)iph);
        else
                return 0;
}

static inline __be32 ip_tunnel_get_flowlabel(const struct iphdr *iph,
                                             const struct sk_buff *skb)
{
        __be16 payload_protocol = skb_protocol(skb, true);

        if (payload_protocol == htons(ETH_P_IPV6))
                return ip6_flowlabel((const struct ipv6hdr *)iph);
        else
                return 0;
}

static inline u8 ip_tunnel_get_ttl(const struct iphdr *iph,
                                       const struct sk_buff *skb)
{
        __be16 payload_protocol = skb_protocol(skb, true);

        if (payload_protocol == htons(ETH_P_IP))
                return iph->ttl;
        else if (payload_protocol == htons(ETH_P_IPV6))
                return ((const struct ipv6hdr *)iph)->hop_limit;
        else
                return 0;
}

/* Propagate ECN bits out */
static inline u8 ip_tunnel_ecn_encap(u8 tos, const struct iphdr *iph,
                                     const struct sk_buff *skb)
{
        u8 inner = ip_tunnel_get_dsfield(iph, skb);

        return INET_ECN_encapsulate(tos, inner);
}

int __iptunnel_pull_header(struct sk_buff *skb, int hdr_len,
                           __be16 inner_proto, bool raw_proto, bool xnet);

static inline int iptunnel_pull_header(struct sk_buff *skb, int hdr_len,
                                       __be16 inner_proto, bool xnet)
{
        return __iptunnel_pull_header(skb, hdr_len, inner_proto, false, xnet);
}

void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb,
                   __be32 src, __be32 dst, u8 proto,
                   u8 tos, u8 ttl, __be16 df, bool xnet);
struct metadata_dst *iptunnel_metadata_reply(struct metadata_dst *md,
                                             gfp_t flags);
int skb_tunnel_check_pmtu(struct sk_buff *skb, struct dst_entry *encap_dst,
                          int headroom, bool reply);

int iptunnel_handle_offloads(struct sk_buff *skb, int gso_type_mask);

static inline int iptunnel_pull_offloads(struct sk_buff *skb)
{
        if (skb_is_gso(skb)) {
                int err;

                err = skb_unclone(skb, GFP_ATOMIC);
                if (unlikely(err))
                        return err;
                skb_shinfo(skb)->gso_type &= ~(NETIF_F_GSO_ENCAP_ALL >>
                                               NETIF_F_GSO_SHIFT);
        }

        skb->encapsulation = 0;
        return 0;
}

static inline void iptunnel_xmit_stats(struct net_device *dev, int pkt_len)
{
        if (pkt_len > 0) {
                struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats);

                u64_stats_update_begin(&tstats->syncp);
                u64_stats_add(&tstats->tx_bytes, pkt_len);
                u64_stats_inc(&tstats->tx_packets);
                u64_stats_update_end(&tstats->syncp);
                put_cpu_ptr(tstats);
                return;
        }

        if (pkt_len < 0) {
                DEV_STATS_INC(dev, tx_errors);
                DEV_STATS_INC(dev, tx_aborted_errors);
        } else {
                DEV_STATS_INC(dev, tx_dropped);
        }
}

static inline void ip_tunnel_info_opts_get(void *to,
                                           const struct ip_tunnel_info *info)
{
        memcpy(to, info + 1, info->options_len);
}

static inline void ip_tunnel_info_opts_set(struct ip_tunnel_info *info,
                                           const void *from, int len,
                                           const unsigned long *flags)
{
        info->options_len = len;
        if (len > 0) {
                memcpy(ip_tunnel_info_opts(info), from, len);
                ip_tunnel_flags_or(info->key.tun_flags, info->key.tun_flags,
                                   flags);
        }
}

static inline struct ip_tunnel_info *lwt_tun_info(struct lwtunnel_state *lwtstate)
{
        return (struct ip_tunnel_info *)lwtstate->data;
}

DECLARE_STATIC_KEY_FALSE(ip_tunnel_metadata_cnt);

/* Returns > 0 if metadata should be collected */
static inline int ip_tunnel_collect_metadata(void)
{
        return static_branch_unlikely(&ip_tunnel_metadata_cnt);
}

void __init ip_tunnel_core_init(void);

void ip_tunnel_need_metadata(void);
void ip_tunnel_unneed_metadata(void);

#else /* CONFIG_INET */

static inline struct ip_tunnel_info *lwt_tun_info(struct lwtunnel_state *lwtstate)
{
        return NULL;
}

static inline void ip_tunnel_need_metadata(void)
{
}

static inline void ip_tunnel_unneed_metadata(void)
{
}

static inline void ip_tunnel_info_opts_get(void *to,
                                           const struct ip_tunnel_info *info)
{
}

static inline void ip_tunnel_info_opts_set(struct ip_tunnel_info *info,
                                           const void *from, int len,
                                           const unsigned long *flags)
{
        info->options_len = 0;
}

#endif /* CONFIG_INET */

#endif /* __NET_IP_TUNNELS_H */




















































































































































































 1115 









  287 














  340 

  340 
  340 

  287 
  309 

  339 


  327 
  261 



   26 


   88 














  379 

  379 
  379 

  340 
  342 

  380 


  356 
  289 



   16 


   77 
   77 
















  403 

  380 






















































  533 

   71 



  531 
  529 



  415 
   64 
  414 






  403 

  402 


  403 
  380 






  513 


  512 














  514 
  496 
  513 


   48 
   48 


  490 





   89 
   89 

   88 


   88 
   69 
   88 


  490 
  490 



  514 




 2139 


 2141 

















 2141 




  837 



  838 

    2 
  836 




   16 



   16 

   16 






   16 




  149 






  552 
  553 
  522 












































































































 1383 




  131 



  131 







































 1326 









 1382 
 1327 





  373 


  373 







   16 


   15 


   16 














   15 

   16 

   14 







   16 

























































































   15 




   15 

    6 
    9 



    9 
    9 

    9 

    9 


    9 
    9 
    9 

    9 
    5 
    5 

    5 


    4 





    4 



    4 
    4 

    4 

    1 

































    1 










    1 
    1 

    1 
    1 

   15 

   14 
   14 
   14 
































    5 
    4 


    5 





    5 
    5 
    5 


    5 










  150 
  148 
    4 



  150 
  128 


  150 
  149 

  150 
















  150 







  151 






  151 
   11 


   11 

   11 



    9 

    1 


    8 
    8 

    2 
    2 

   10 
    3 
   10 




  151 







  151 

    5 
    5 



  150 
  150 

  127 
  127 



  150 









   66 

  172 

  230 


  974 



  974 









  151 














  149 









  151 











  150 






  151 











  151 

  149 
  149 


  150 

  148 


  151 
   15 






   15 

   15 
   15 












   12 
  151 






   10 








    1 



  151 
  151 



  151 
  151 
  151 









  151 

  151 
    5 


  151 






  151 


  151 









  151 




  151 


  151 

  151 
  151 
  151 

   19 
   19 



   18 



  151 

  151 


  151 








  151 




  151 


  151 

  151 
  151 











  151 

  151 


  150 













  151 




  151 

  151 























  160 


  160 
















  160 







  160 
  159 

  151 
   10 

  150 















  151 


  150 

  151 







  151 
  151 



  151 


  150 


  151 





  150 

  150 
  150 
  150 








  388 



  389 







  389 



   17 




    3 


    1 






































  136 
  131 
   85 





  136 
  130 
  136 


  276 
  275 









  335 

  336 


  389 


















  396 







   15 
   15 
  210 



   15 



  389 
  389 






  389 
   19 


   19 




  385 










  433 













  431 


  432 








  409 


  408 
   26 
  428 









   48 

   48 















   48 
   30 


   29 

   30 

   19 
   16 
   15 

   16 








    3 





    3 










   21 

  433 

  433 



  433 
   85 
   85 

  433 







  433 
   85 












  471 

  472 
  472 
   43 
    6 
   37 




  467 
    5 









  469 



  433 

  123 
  472 












  521 

  520 
  522 







  520 

  472 

  472 
  521 















  526 

  522 













  526 
  526 
  526 

  525 


  526 
  526 
  525 
  525 







  546 




  549 



  547 
  256 

  548 


  549 
  546 











   52 

   44 
   44 



  526 





























  515 








  517 

  516 

   48 
  517 

   47 
  516 
   96 
  514 














   43 






    4 
   43 
   36 
   43 





   43 
   43 
    4 































  139 



  141 


  141 



  141 






  131 



  132 



































    9 

    9 

    9 


    9 

    9 
    9 








    9 


    9 


    9 
    9 


    9 
    9 


    9 
    9 






























    9 


















    9 



    9 




    1 


    9 

    9 






    9 
    9 
    9 






    9 


    9 



    9 



    9 
   10 




















   10 

   10 

    9 
    9 
    9 
    9 


   10 


































































































































































































































































































































































































































































































































































































































  394 


  394 
  396 

  394 











  396 

  393 
  394 



  395 





  394 
















  397 



  394 


  397 
  395 

  397 
  394 
  394 




  397 



  396 














  395 






  397 
  394 

  395 

  395 




  397 



  389 













  394 

  393 






  395 



  393 




  397 














  396 






  395 

  395 









  397 



  394 














  393 



































  212 





















  169 



























































































 2028 

















































  339 

  338 

  339 









  337 



  339 










  319 


  321 
  320 

  313 




  339 













   31 



   31 

   31 
   31 






   31 




   31 


   31 










 1405 

  357 

    1 




















 1058 


 1057 

  630 
    7 


  629 


    7 






















  231 












  231 
  169 
  231 
  228 


  231 
  231 



  230 


  169 


















  231 




  231 





  231 
  231 
  169 
    2 



   66 
   66 


  231 


    6 
    6 
    6 


  226 









  230 


    6 
  231 
























  169 





  231 


    3 


  231 

  231 
  169 
  229 
  169 
  169 
  169 


  231 
















































































    2 




























  245 














  181 









  176 

  174 




    5 
  174 

   34 

   34 
    4 
   34 
    4 







   30 





























  245 





  243 





























  245 


  183 





  243 









  245 









  183 
  181 
   32 
   30 
   32 
    2 


   31 






  169 

  231 

  169 


  231 


















   28 
























    5 






    5 


    5 





    5 

    5 

























   51 

   30 

   50 























   50 










   51 




















































  186 

  186 
  184 







  184 
   69 
























 2907 
 1263 

 2914 




















    5 
    5 









    5 



    5 











































































































































































































  212 
















  212 


  212 
   24 


   19 
    5 


    5 




























    5 
    5 








  189 



  189 

   90 



  184 












































  184 









    1 

    1 







  181 











  187 



  187 







  187 








  187 




  187 




  187 








  187 
   72 
   35 


  187 




  187 

  187 


  186 
























  186 





































  186 
  186 





  186 
  186 






  180 
    1 



























  186 







  186 
   71 

  186 

  186 
  186 
  186 
  186 








  186 
    7 
  182 
  180 
  180 

   70 





  186 

  186 





  186 






  184 
  184 
  184 

    9 



  186 

  186 



  186 













  186 
    3 








  185 
  185 


  186 




  185 




    9 

    9 

   20 
   20 



   20 





  187 







    1 
    1 








  828 




















  830 

  829 
























































  830 

















 1260 






 1258 



 1256 

  464 

  462 



  461 



  457 



  460 







  830 
  828 


  830 





  828 
  829 








  830 

  828 
  829 

  829 


  830 







  829 




  827 





  827 


  827 




 1259 




 1258 
 1263 


    6 












 1717 


















 1341 
  282 




 1718 




 1702 















 1697 
    3 

 1721 




















  122 
  122 
  117 
  120 









  122 
  122 

  121 


  117 

  117 


























  117 

    5 

  117 








  117 







  117 




















 1943 





 1947 






 1942 
  338 



  337 
    2 
    2 


 1942 

 1947 







 2810 


 1970 



















 1698 



  341 






    2 

 1697 






    8 




 1700 
 1394 
   69 




 1333 
 1333 

  587 


 1651 
 1639 






 1649 

 1638 
   16 









   12 






    9 




 1648 


 1701 


 1646 



    1 





 1644 

 1640 
 1644 



 1646 

































































 1715 















 1120 




 1717 
 1713 


 1717 








 1721 


 1716 



 1715 












 1726 
 1717 




 1415 



 1414 



 1401 
 1676 
 1396 

   16 









    7 

    6 
    6 


    6 



    6 




    2 


    2 



    2 









    5 









  340 



  340 



  339 





  339 









  339 


    9 
   24 



  339 














 2019 






    2 











    1 




    1 

 2020 
 1994 
  344 
    7 

  351 


 2017 
   53 
 2906 








    1 
















    1 





    1 


    1 


    1 

    1 

    1 

    1 
    1 



    1 








    1 
    1 

    1 
    1 
    1 
















































    1 
    1 

















    1 





    1 
    1 
    1 


    1 
    1 


    1 
    1 




































    1 



    1 

    1 


    1 





   10 

   10 








   10 
   10 



   10 





   10 












































 3127 





























 3126 






 2023 


 1862 



















 1864 


  439 
 1620 
 1616 




 2909 

  437 
  206 

  264 
    1 

  264 





  264 
  252 
  243 
   13 
   12 

   27 



















   90 












 3139 





 2028 












 3139 



 3133 
  986 







 3131 
 3130 
















 3139 




 3139 


 3134 
 2034 

   10 



 3126 

 1870 
    3 


    3 


 1869 
   15 


   15 
   12 
   10 
   18 


    6 





 3131 
























 3257 







 3256 
















 3233 


  204 

 3155 










  200 

 2599 






 3266 




 3256 















    7 





    7 

 3249 

 1419 


 1421 








 3255 


















 3256 





 3266 



 3266 






 3259 






  465 

 3266 


  163 

 3126 







 3256 



   21 

 3257 
  464 










 3265 

 3255 








 2957 


  120 
  117 




  120 

















   24 
   24 
   23 



   24 


























 2957 


 2937 
 2955 






  663 
  652 













   24 


   24 


   24 

   24 



   24 



   11 



   13 













  470 


  470 

  471 



  468 



  468 













  468 


  468 





   12 



































  227 



  225 

  227 

  227 


  226 
  227 











 1214 



 1210 
 1209 
 1211 





 1211 
 1210 



































































































































































































































   14 
   14 


   14 


   14 



   14 



   14 




    3 

    2 



    2 











    1 







   14 

    5 

   14 

   14 



   14 


   14 

   14 



   14 

   14 































   14 



   14 



   14 






























 10523 

 10519 

 10513 


 10523 

















    7 
  159 


  156 






  159 
  158 

    7 




    5 



    5 









  159 
   13 
   13 





    3 















  154 


















  156 


  154 

  156 
































    7 



    7 






    7 









    7 
















































 2238 


 2237 
 2188 




  346 





  814 
   35 




  815 
   35 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/mm/memory.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 */

/*
 * demand-loading started 01.12.91 - seems it is high on the list of
 * things wanted, and it should be easy to implement. - Linus
 */

/*
 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
 * pages started 02.12.91, seems to work. - Linus.
 *
 * Tested sharing by executing about 30 /bin/sh: under the old kernel it
 * would have taken more than the 6M I have free, but it worked well as
 * far as I could see.
 *
 * Also corrected some "invalidate()"s - I wasn't doing enough of them.
 */

/*
 * Real VM (paging to/from disk) started 18.12.91. Much more work and
 * thought has to go into this. Oh, well..
 * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why.
 *                Found it. Everything seems to work now.
 * 20.12.91  -  Ok, making the swap-device changeable like the root.
 */

/*
 * 05.04.94  -  Multi-page memory management added for v1.1.
 *              Idea by Alex Bligh (alex@cconcepts.co.uk)
 *
 * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG
 *                (Gerhard.Wichert@pdb.siemens.de)
 *
 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
 */

#include <linux/kernel_stat.h>
#include <linux/mm.h>
#include <linux/mm_inline.h>
#include <linux/sched/mm.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/task.h>
#include <linux/hugetlb.h>
#include <linux/mman.h>
#include <linux/swap.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/memremap.h>
#include <linux/kmsan.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/export.h>
#include <linux/delayacct.h>
#include <linux/init.h>
#include <linux/pfn_t.h>
#include <linux/writeback.h>
#include <linux/memcontrol.h>
#include <linux/mmu_notifier.h>
#include <linux/swapops.h>
#include <linux/elf.h>
#include <linux/gfp.h>
#include <linux/migrate.h>
#include <linux/string.h>
#include <linux/memory-tiers.h>
#include <linux/debugfs.h>
#include <linux/userfaultfd_k.h>
#include <linux/dax.h>
#include <linux/oom.h>
#include <linux/numa.h>
#include <linux/perf_event.h>
#include <linux/ptrace.h>
#include <linux/vmalloc.h>
#include <linux/sched/sysctl.h>

#include <trace/events/kmem.h>

#include <asm/io.h>
#include <asm/mmu_context.h>
#include <asm/pgalloc.h>
#include <linux/uaccess.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>

#include "pgalloc-track.h"
#include "internal.h"
#include "swap.h"

#if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
#endif

#ifndef CONFIG_NUMA
unsigned long max_mapnr;
EXPORT_SYMBOL(max_mapnr);

struct page *mem_map;
EXPORT_SYMBOL(mem_map);
#endif

static vm_fault_t do_fault(struct vm_fault *vmf);
static vm_fault_t do_anonymous_page(struct vm_fault *vmf);
static bool vmf_pte_changed(struct vm_fault *vmf);

/*
 * Return true if the original pte was a uffd-wp pte marker (so the pte was
 * wr-protected).
 */
static __always_inline bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)
{
        if (!userfaultfd_wp(vmf->vma))
                return false;
        if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
                return false;

        return pte_marker_uffd_wp(vmf->orig_pte);
}

/*
 * A number of key systems in x86 including ioremap() rely on the assumption
 * that high_memory defines the upper bound on direct map memory, then end
 * of ZONE_NORMAL.
 */
void *high_memory;
EXPORT_SYMBOL(high_memory);

/*
 * Randomize the address space (stacks, mmaps, brk, etc.).
 *
 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
 *   as ancient (libc5 based) binaries can segfault. )
 */
int randomize_va_space __read_mostly =
#ifdef CONFIG_COMPAT_BRK
                                        1;
#else
                                        2;
#endif

#ifndef arch_wants_old_prefaulted_pte
static inline bool arch_wants_old_prefaulted_pte(void)
{
        /*
         * Transitioning a PTE from 'old' to 'young' can be expensive on
         * some architectures, even if it's performed in hardware. By
         * default, "false" means prefaulted entries will be 'young'.
         */
        return false;
}
#endif

static int __init disable_randmaps(char *s)
{
        randomize_va_space = 0;
        return 1;
}
__setup("norandmaps", disable_randmaps);

unsigned long zero_pfn __read_mostly;
EXPORT_SYMBOL(zero_pfn);

unsigned long highest_memmap_pfn __read_mostly;

/*
 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
 */
static int __init init_zero_pfn(void)
{
        zero_pfn = page_to_pfn(ZERO_PAGE(0));
        return 0;
}
early_initcall(init_zero_pfn);

void mm_trace_rss_stat(struct mm_struct *mm, int member)
{
        trace_rss_stat(mm, member);
}

/*
 * Note: this doesn't free the actual pages themselves. That
 * has been handled earlier when unmapping all the memory regions.
 */
static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
                           unsigned long addr)
{
        pgtable_t token = pmd_pgtable(*pmd);
        pmd_clear(pmd);
        pte_free_tlb(tlb, token, addr);
        mm_dec_nr_ptes(tlb->mm);
}

static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
                                unsigned long addr, unsigned long end,
                                unsigned long floor, unsigned long ceiling)
{
        pmd_t *pmd;
        unsigned long next;
        unsigned long start;

        start = addr;
        pmd = pmd_offset(pud, addr);
        do {
                next = pmd_addr_end(addr, end);
                if (pmd_none_or_clear_bad(pmd))
                        continue;
                free_pte_range(tlb, pmd, addr);
        } while (pmd++, addr = next, addr != end);

        start &= PUD_MASK;
        if (start < floor)
                return;
        if (ceiling) {
                ceiling &= PUD_MASK;
                if (!ceiling)
                        return;
        }
        if (end - 1 > ceiling - 1)
                return;

        pmd = pmd_offset(pud, start);
        pud_clear(pud);
        pmd_free_tlb(tlb, pmd, start);
        mm_dec_nr_pmds(tlb->mm);
}

static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
                                unsigned long addr, unsigned long end,
                                unsigned long floor, unsigned long ceiling)
{
        pud_t *pud;
        unsigned long next;
        unsigned long start;

        start = addr;
        pud = pud_offset(p4d, addr);
        do {
                next = pud_addr_end(addr, end);
                if (pud_none_or_clear_bad(pud))
                        continue;
                free_pmd_range(tlb, pud, addr, next, floor, ceiling);
        } while (pud++, addr = next, addr != end);

        start &= P4D_MASK;
        if (start < floor)
                return;
        if (ceiling) {
                ceiling &= P4D_MASK;
                if (!ceiling)
                        return;
        }
        if (end - 1 > ceiling - 1)
                return;

        pud = pud_offset(p4d, start);
        p4d_clear(p4d);
        pud_free_tlb(tlb, pud, start);
        mm_dec_nr_puds(tlb->mm);
}

static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
                                unsigned long addr, unsigned long end,
                                unsigned long floor, unsigned long ceiling)
{
        p4d_t *p4d;
        unsigned long next;
        unsigned long start;

        start = addr;
        p4d = p4d_offset(pgd, addr);
        do {
                next = p4d_addr_end(addr, end);
                if (p4d_none_or_clear_bad(p4d))
                        continue;
                free_pud_range(tlb, p4d, addr, next, floor, ceiling);
        } while (p4d++, addr = next, addr != end);

        start &= PGDIR_MASK;
        if (start < floor)
                return;
        if (ceiling) {
                ceiling &= PGDIR_MASK;
                if (!ceiling)
                        return;
        }
        if (end - 1 > ceiling - 1)
                return;

        p4d = p4d_offset(pgd, start);
        pgd_clear(pgd);
        p4d_free_tlb(tlb, p4d, start);
}

/*
 * This function frees user-level page tables of a process.
 */
void free_pgd_range(struct mmu_gather *tlb,
                        unsigned long addr, unsigned long end,
                        unsigned long floor, unsigned long ceiling)
{
        pgd_t *pgd;
        unsigned long next;

        /*
         * The next few lines have given us lots of grief...
         *
         * Why are we testing PMD* at this top level?  Because often
         * there will be no work to do at all, and we'd prefer not to
         * go all the way down to the bottom just to discover that.
         *
         * Why all these "- 1"s?  Because 0 represents both the bottom
         * of the address space and the top of it (using -1 for the
         * top wouldn't help much: the masks would do the wrong thing).
         * The rule is that addr 0 and floor 0 refer to the bottom of
         * the address space, but end 0 and ceiling 0 refer to the top
         * Comparisons need to use "end - 1" and "ceiling - 1" (though
         * that end 0 case should be mythical).
         *
         * Wherever addr is brought up or ceiling brought down, we must
         * be careful to reject "the opposite 0" before it confuses the
         * subsequent tests.  But what about where end is brought down
         * by PMD_SIZE below? no, end can't go down to 0 there.
         *
         * Whereas we round start (addr) and ceiling down, by different
         * masks at different levels, in order to test whether a table
         * now has no other vmas using it, so can be freed, we don't
         * bother to round floor or end up - the tests don't need that.
         */

        addr &= PMD_MASK;
        if (addr < floor) {
                addr += PMD_SIZE;
                if (!addr)
                        return;
        }
        if (ceiling) {
                ceiling &= PMD_MASK;
                if (!ceiling)
                        return;
        }
        if (end - 1 > ceiling - 1)
                end -= PMD_SIZE;
        if (addr > end - 1)
                return;
        /*
         * We add page table cache pages with PAGE_SIZE,
         * (see pte_free_tlb()), flush the tlb if we need
         */
        tlb_change_page_size(tlb, PAGE_SIZE);
        pgd = pgd_offset(tlb->mm, addr);
        do {
                next = pgd_addr_end(addr, end);
                if (pgd_none_or_clear_bad(pgd))
                        continue;
                free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
        } while (pgd++, addr = next, addr != end);
}

void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
                   struct vm_area_struct *vma, unsigned long floor,
                   unsigned long ceiling, bool mm_wr_locked)
{
        struct unlink_vma_file_batch vb;

        do {
                unsigned long addr = vma->vm_start;
                struct vm_area_struct *next;

                /*
                 * Note: USER_PGTABLES_CEILING may be passed as ceiling and may
                 * be 0.  This will underflow and is okay.
                 */
                next = mas_find(mas, ceiling - 1);
                if (unlikely(xa_is_zero(next)))
                        next = NULL;

                /*
                 * Hide vma from rmap and truncate_pagecache before freeing
                 * pgtables
                 */
                if (mm_wr_locked)
                        vma_start_write(vma);
                unlink_anon_vmas(vma);

                if (is_vm_hugetlb_page(vma)) {
                        unlink_file_vma(vma);
                        hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
                                floor, next ? next->vm_start : ceiling);
                } else {
                        unlink_file_vma_batch_init(&vb);
                        unlink_file_vma_batch_add(&vb, vma);

                        /*
                         * Optimization: gather nearby vmas into one call down
                         */
                        while (next && next->vm_start <= vma->vm_end + PMD_SIZE
                               && !is_vm_hugetlb_page(next)) {
                                vma = next;
                                next = mas_find(mas, ceiling - 1);
                                if (unlikely(xa_is_zero(next)))
                                        next = NULL;
                                if (mm_wr_locked)
                                        vma_start_write(vma);
                                unlink_anon_vmas(vma);
                                unlink_file_vma_batch_add(&vb, vma);
                        }
                        unlink_file_vma_batch_final(&vb);
                        free_pgd_range(tlb, addr, vma->vm_end,
                                floor, next ? next->vm_start : ceiling);
                }
                vma = next;
        } while (vma);
}

void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte)
{
        spinlock_t *ptl = pmd_lock(mm, pmd);

        if (likely(pmd_none(*pmd))) {        /* Has another populated it ? */
                mm_inc_nr_ptes(mm);
                /*
                 * Ensure all pte setup (eg. pte page lock and page clearing) are
                 * visible before the pte is made visible to other CPUs by being
                 * put into page tables.
                 *
                 * The other side of the story is the pointer chasing in the page
                 * table walking code (when walking the page table without locking;
                 * ie. most of the time). Fortunately, these data accesses consist
                 * of a chain of data-dependent loads, meaning most CPUs (alpha
                 * being the notable exception) will already guarantee loads are
                 * seen in-order. See the alpha page table accessors for the
                 * smp_rmb() barriers in page table walking code.
                 */
                smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
                pmd_populate(mm, pmd, *pte);
                *pte = NULL;
        }
        spin_unlock(ptl);
}

int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
{
        pgtable_t new = pte_alloc_one(mm);
        if (!new)
                return -ENOMEM;

        pmd_install(mm, pmd, &new);
        if (new)
                pte_free(mm, new);
        return 0;
}

int __pte_alloc_kernel(pmd_t *pmd)
{
        pte_t *new = pte_alloc_one_kernel(&init_mm);
        if (!new)
                return -ENOMEM;

        spin_lock(&init_mm.page_table_lock);
        if (likely(pmd_none(*pmd))) {        /* Has another populated it ? */
                smp_wmb(); /* See comment in pmd_install() */
                pmd_populate_kernel(&init_mm, pmd, new);
                new = NULL;
        }
        spin_unlock(&init_mm.page_table_lock);
        if (new)
                pte_free_kernel(&init_mm, new);
        return 0;
}

static inline void init_rss_vec(int *rss)
{
        memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
}

static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
{
        int i;

        for (i = 0; i < NR_MM_COUNTERS; i++)
                if (rss[i])
                        add_mm_counter(mm, i, rss[i]);
}

/*
 * This function is called to print an error when a bad pte
 * is found. For example, we might have a PFN-mapped pte in
 * a region that doesn't allow it.
 *
 * The calling function must still handle the error.
 */
static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
                          pte_t pte, struct page *page)
{
        pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
        p4d_t *p4d = p4d_offset(pgd, addr);
        pud_t *pud = pud_offset(p4d, addr);
        pmd_t *pmd = pmd_offset(pud, addr);
        struct address_space *mapping;
        pgoff_t index;
        static unsigned long resume;
        static unsigned long nr_shown;
        static unsigned long nr_unshown;

        /*
         * Allow a burst of 60 reports, then keep quiet for that minute;
         * or allow a steady drip of one report per second.
         */
        if (nr_shown == 60) {
                if (time_before(jiffies, resume)) {
                        nr_unshown++;
                        return;
                }
                if (nr_unshown) {
                        pr_alert("BUG: Bad page map: %lu messages suppressed\n",
                                 nr_unshown);
                        nr_unshown = 0;
                }
                nr_shown = 0;
        }
        if (nr_shown++ == 0)
                resume = jiffies + 60 * HZ;

        mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
        index = linear_page_index(vma, addr);

        pr_alert("BUG: Bad page map in process %s  pte:%08llx pmd:%08llx\n",
                 current->comm,
                 (long long)pte_val(pte), (long long)pmd_val(*pmd));
        if (page)
                dump_page(page, "bad pte");
        pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
                 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
        pr_alert("file:%pD fault:%ps mmap:%ps read_folio:%ps\n",
                 vma->vm_file,
                 vma->vm_ops ? vma->vm_ops->fault : NULL,
                 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
                 mapping ? mapping->a_ops->read_folio : NULL);
        dump_stack();
        add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
}

/*
 * vm_normal_page -- This function gets the "struct page" associated with a pte.
 *
 * "Special" mappings do not wish to be associated with a "struct page" (either
 * it doesn't exist, or it exists but they don't want to touch it). In this
 * case, NULL is returned here. "Normal" mappings do have a struct page.
 *
 * There are 2 broad cases. Firstly, an architecture may define a pte_special()
 * pte bit, in which case this function is trivial. Secondly, an architecture
 * may not have a spare pte bit, which requires a more complicated scheme,
 * described below.
 *
 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
 * special mapping (even if there are underlying and valid "struct pages").
 * COWed pages of a VM_PFNMAP are always normal.
 *
 * The way we recognize COWed pages within VM_PFNMAP mappings is through the
 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
 * set, and the vm_pgoff will point to the first PFN mapped: thus every special
 * mapping will always honor the rule
 *
 *        pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
 *
 * And for normal mappings this is false.
 *
 * This restricts such mappings to be a linear translation from virtual address
 * to pfn. To get around this restriction, we allow arbitrary mappings so long
 * as the vma is not a COW mapping; in that case, we know that all ptes are
 * special (because none can have been COWed).
 *
 *
 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
 *
 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
 * page" backing, however the difference is that _all_ pages with a struct
 * page (that is, those where pfn_valid is true) are refcounted and considered
 * normal pages by the VM. The only exception are zeropages, which are
 * *never* refcounted.
 *
 * The disadvantage is that pages are refcounted (which can be slower and
 * simply not an option for some PFNMAP users). The advantage is that we
 * don't have to follow the strict linearity rule of PFNMAP mappings in
 * order to support COWable mappings.
 *
 */
struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
                            pte_t pte)
{
        unsigned long pfn = pte_pfn(pte);

        if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
                if (likely(!pte_special(pte)))
                        goto check_pfn;
                if (vma->vm_ops && vma->vm_ops->find_special_page)
                        return vma->vm_ops->find_special_page(vma, addr);
                if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
                        return NULL;
                if (is_zero_pfn(pfn))
                        return NULL;
                if (pte_devmap(pte))
                /*
                 * NOTE: New users of ZONE_DEVICE will not set pte_devmap()
                 * and will have refcounts incremented on their struct pages
                 * when they are inserted into PTEs, thus they are safe to
                 * return here. Legacy ZONE_DEVICE pages that set pte_devmap()
                 * do not have refcounts. Example of legacy ZONE_DEVICE is
                 * MEMORY_DEVICE_FS_DAX type in pmem or virtio_fs drivers.
                 */
                        return NULL;

                print_bad_pte(vma, addr, pte, NULL);
                return NULL;
        }

        /* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */

        if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
                if (vma->vm_flags & VM_MIXEDMAP) {
                        if (!pfn_valid(pfn))
                                return NULL;
                        if (is_zero_pfn(pfn))
                                return NULL;
                        goto out;
                } else {
                        unsigned long off;
                        off = (addr - vma->vm_start) >> PAGE_SHIFT;
                        if (pfn == vma->vm_pgoff + off)
                                return NULL;
                        if (!is_cow_mapping(vma->vm_flags))
                                return NULL;
                }
        }

        if (is_zero_pfn(pfn))
                return NULL;

check_pfn:
        if (unlikely(pfn > highest_memmap_pfn)) {
                print_bad_pte(vma, addr, pte, NULL);
                return NULL;
        }

        /*
         * NOTE! We still have PageReserved() pages in the page tables.
         * eg. VDSO mappings can cause them to exist.
         */
out:
        VM_WARN_ON_ONCE(is_zero_pfn(pfn));
        return pfn_to_page(pfn);
}

struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
                            pte_t pte)
{
        struct page *page = vm_normal_page(vma, addr, pte);

        if (page)
                return page_folio(page);
        return NULL;
}

#ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES
struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
                                pmd_t pmd)
{
        unsigned long pfn = pmd_pfn(pmd);

        /* Currently it's only used for huge pfnmaps */
        if (unlikely(pmd_special(pmd)))
                return NULL;

        if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
                if (vma->vm_flags & VM_MIXEDMAP) {
                        if (!pfn_valid(pfn))
                                return NULL;
                        goto out;
                } else {
                        unsigned long off;
                        off = (addr - vma->vm_start) >> PAGE_SHIFT;
                        if (pfn == vma->vm_pgoff + off)
                                return NULL;
                        if (!is_cow_mapping(vma->vm_flags))
                                return NULL;
                }
        }

        if (pmd_devmap(pmd))
                return NULL;
        if (is_huge_zero_pmd(pmd))
                return NULL;
        if (unlikely(pfn > highest_memmap_pfn))
                return NULL;

        /*
         * NOTE! We still have PageReserved() pages in the page tables.
         * eg. VDSO mappings can cause them to exist.
         */
out:
        return pfn_to_page(pfn);
}

struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
                                  unsigned long addr, pmd_t pmd)
{
        struct page *page = vm_normal_page_pmd(vma, addr, pmd);

        if (page)
                return page_folio(page);
        return NULL;
}
#endif

static void restore_exclusive_pte(struct vm_area_struct *vma,
                                  struct page *page, unsigned long address,
                                  pte_t *ptep)
{
        struct folio *folio = page_folio(page);
        pte_t orig_pte;
        pte_t pte;
        swp_entry_t entry;

        orig_pte = ptep_get(ptep);
        pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
        if (pte_swp_soft_dirty(orig_pte))
                pte = pte_mksoft_dirty(pte);

        entry = pte_to_swp_entry(orig_pte);
        if (pte_swp_uffd_wp(orig_pte))
                pte = pte_mkuffd_wp(pte);
        else if (is_writable_device_exclusive_entry(entry))
                pte = maybe_mkwrite(pte_mkdirty(pte), vma);

        VM_BUG_ON_FOLIO(pte_write(pte) && (!folio_test_anon(folio) &&
                                           PageAnonExclusive(page)), folio);

        /*
         * No need to take a page reference as one was already
         * created when the swap entry was made.
         */
        if (folio_test_anon(folio))
                folio_add_anon_rmap_pte(folio, page, vma, address, RMAP_NONE);
        else
                /*
                 * Currently device exclusive access only supports anonymous
                 * memory so the entry shouldn't point to a filebacked page.
                 */
                WARN_ON_ONCE(1);

        set_pte_at(vma->vm_mm, address, ptep, pte);

        /*
         * No need to invalidate - it was non-present before. However
         * secondary CPUs may have mappings that need invalidating.
         */
        update_mmu_cache(vma, address, ptep);
}

/*
 * Tries to restore an exclusive pte if the page lock can be acquired without
 * sleeping.
 */
static int
try_restore_exclusive_pte(pte_t *src_pte, struct vm_area_struct *vma,
                        unsigned long addr)
{
        swp_entry_t entry = pte_to_swp_entry(ptep_get(src_pte));
        struct page *page = pfn_swap_entry_to_page(entry);

        if (trylock_page(page)) {
                restore_exclusive_pte(vma, page, addr, src_pte);
                unlock_page(page);
                return 0;
        }

        return -EBUSY;
}

/*
 * copy one vm_area from one task to the other. Assumes the page tables
 * already present in the new task to be cleared in the whole range
 * covered by this vma.
 */

static unsigned long
copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
                struct vm_area_struct *src_vma, unsigned long addr, int *rss)
{
        unsigned long vm_flags = dst_vma->vm_flags;
        pte_t orig_pte = ptep_get(src_pte);
        pte_t pte = orig_pte;
        struct folio *folio;
        struct page *page;
        swp_entry_t entry = pte_to_swp_entry(orig_pte);

        if (likely(!non_swap_entry(entry))) {
                if (swap_duplicate(entry) < 0)
                        return -EIO;

                /* make sure dst_mm is on swapoff's mmlist. */
                if (unlikely(list_empty(&dst_mm->mmlist))) {
                        spin_lock(&mmlist_lock);
                        if (list_empty(&dst_mm->mmlist))
                                list_add(&dst_mm->mmlist,
                                                &src_mm->mmlist);
                        spin_unlock(&mmlist_lock);
                }
                /* Mark the swap entry as shared. */
                if (pte_swp_exclusive(orig_pte)) {
                        pte = pte_swp_clear_exclusive(orig_pte);
                        set_pte_at(src_mm, addr, src_pte, pte);
                }
                rss[MM_SWAPENTS]++;
        } else if (is_migration_entry(entry)) {
                folio = pfn_swap_entry_folio(entry);

                rss[mm_counter(folio)]++;

                if (!is_readable_migration_entry(entry) &&
                                is_cow_mapping(vm_flags)) {
                        /*
                         * COW mappings require pages in both parent and child
                         * to be set to read. A previously exclusive entry is
                         * now shared.
                         */
                        entry = make_readable_migration_entry(
                                                        swp_offset(entry));
                        pte = swp_entry_to_pte(entry);
                        if (pte_swp_soft_dirty(orig_pte))
                                pte = pte_swp_mksoft_dirty(pte);
                        if (pte_swp_uffd_wp(orig_pte))
                                pte = pte_swp_mkuffd_wp(pte);
                        set_pte_at(src_mm, addr, src_pte, pte);
                }
        } else if (is_device_private_entry(entry)) {
                page = pfn_swap_entry_to_page(entry);
                folio = page_folio(page);

                /*
                 * Update rss count even for unaddressable pages, as
                 * they should treated just like normal pages in this
                 * respect.
                 *
                 * We will likely want to have some new rss counters
                 * for unaddressable pages, at some point. But for now
                 * keep things as they are.
                 */
                folio_get(folio);
                rss[mm_counter(folio)]++;
                /* Cannot fail as these pages cannot get pinned. */
                folio_try_dup_anon_rmap_pte(folio, page, src_vma);

                /*
                 * We do not preserve soft-dirty information, because so
                 * far, checkpoint/restore is the only feature that
                 * requires that. And checkpoint/restore does not work
                 * when a device driver is involved (you cannot easily
                 * save and restore device driver state).
                 */
                if (is_writable_device_private_entry(entry) &&
                    is_cow_mapping(vm_flags)) {
                        entry = make_readable_device_private_entry(
                                                        swp_offset(entry));
                        pte = swp_entry_to_pte(entry);
                        if (pte_swp_uffd_wp(orig_pte))
                                pte = pte_swp_mkuffd_wp(pte);
                        set_pte_at(src_mm, addr, src_pte, pte);
                }
        } else if (is_device_exclusive_entry(entry)) {
                /*
                 * Make device exclusive entries present by restoring the
                 * original entry then copying as for a present pte. Device
                 * exclusive entries currently only support private writable
                 * (ie. COW) mappings.
                 */
                VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
                if (try_restore_exclusive_pte(src_pte, src_vma, addr))
                        return -EBUSY;
                return -ENOENT;
        } else if (is_pte_marker_entry(entry)) {
                pte_marker marker = copy_pte_marker(entry, dst_vma);

                if (marker)
                        set_pte_at(dst_mm, addr, dst_pte,
                                   make_pte_marker(marker));
                return 0;
        }
        if (!userfaultfd_wp(dst_vma))
                pte = pte_swp_clear_uffd_wp(pte);
        set_pte_at(dst_mm, addr, dst_pte, pte);
        return 0;
}

/*
 * Copy a present and normal page.
 *
 * NOTE! The usual case is that this isn't required;
 * instead, the caller can just increase the page refcount
 * and re-use the pte the traditional way.
 *
 * And if we need a pre-allocated page but don't yet have
 * one, return a negative error to let the preallocation
 * code know so that it can do so outside the page table
 * lock.
 */
static inline int
copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
                  pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
                  struct folio **prealloc, struct page *page)
{
        struct folio *new_folio;
        pte_t pte;

        new_folio = *prealloc;
        if (!new_folio)
                return -EAGAIN;

        /*
         * We have a prealloc page, all good!  Take it
         * over and copy the page & arm it.
         */

        if (copy_mc_user_highpage(&new_folio->page, page, addr, src_vma))
                return -EHWPOISON;

        *prealloc = NULL;
        __folio_mark_uptodate(new_folio);
        folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE);
        folio_add_lru_vma(new_folio, dst_vma);
        rss[MM_ANONPAGES]++;

        /* All done, just insert the new page copy in the child */
        pte = mk_pte(&new_folio->page, dst_vma->vm_page_prot);
        pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
        if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
                /* Uffd-wp needs to be delivered to dest pte as well */
                pte = pte_mkuffd_wp(pte);
        set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
        return 0;
}

static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
                struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte,
                pte_t pte, unsigned long addr, int nr)
{
        struct mm_struct *src_mm = src_vma->vm_mm;

        /* If it's a COW mapping, write protect it both processes. */
        if (is_cow_mapping(src_vma->vm_flags) && pte_write(pte)) {
                wrprotect_ptes(src_mm, addr, src_pte, nr);
                pte = pte_wrprotect(pte);
        }

        /* If it's a shared mapping, mark it clean in the child. */
        if (src_vma->vm_flags & VM_SHARED)
                pte = pte_mkclean(pte);
        pte = pte_mkold(pte);

        if (!userfaultfd_wp(dst_vma))
                pte = pte_clear_uffd_wp(pte);

        set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);
}

/*
 * Copy one present PTE, trying to batch-process subsequent PTEs that map
 * consecutive pages of the same folio by copying them as well.
 *
 * Returns -EAGAIN if one preallocated page is required to copy the next PTE.
 * Otherwise, returns the number of copied PTEs (at least 1).
 */
static inline int
copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
                 pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr,
                 int max_nr, int *rss, struct folio **prealloc)
{
        struct page *page;
        struct folio *folio;
        bool any_writable;
        fpb_t flags = 0;
        int err, nr;

        page = vm_normal_page(src_vma, addr, pte);
        if (unlikely(!page))
                goto copy_pte;

        folio = page_folio(page);

        /*
         * If we likely have to copy, just don't bother with batching. Make
         * sure that the common "small folio" case is as fast as possible
         * by keeping the batching logic separate.
         */
        if (unlikely(!*prealloc && folio_test_large(folio) && max_nr != 1)) {
                if (src_vma->vm_flags & VM_SHARED)
                        flags |= FPB_IGNORE_DIRTY;
                if (!vma_soft_dirty_enabled(src_vma))
                        flags |= FPB_IGNORE_SOFT_DIRTY;

                nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr, flags,
                                     &any_writable, NULL, NULL);
                folio_ref_add(folio, nr);
                if (folio_test_anon(folio)) {
                        if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page,
                                                                  nr, src_vma))) {
                                folio_ref_sub(folio, nr);
                                return -EAGAIN;
                        }
                        rss[MM_ANONPAGES] += nr;
                        VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio);
                } else {
                        folio_dup_file_rmap_ptes(folio, page, nr);
                        rss[mm_counter_file(folio)] += nr;
                }
                if (any_writable)
                        pte = pte_mkwrite(pte, src_vma);
                __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte,
                                    addr, nr);
                return nr;
        }

        folio_get(folio);
        if (folio_test_anon(folio)) {
                /*
                 * If this page may have been pinned by the parent process,
                 * copy the page immediately for the child so that we'll always
                 * guarantee the pinned page won't be randomly replaced in the
                 * future.
                 */
                if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, src_vma))) {
                        /* Page may be pinned, we have to copy. */
                        folio_put(folio);
                        err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
                                                addr, rss, prealloc, page);
                        return err ? err : 1;
                }
                rss[MM_ANONPAGES]++;
                VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio);
        } else {
                folio_dup_file_rmap_pte(folio, page);
                rss[mm_counter_file(folio)]++;
        }

copy_pte:
        __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, 1);
        return 1;
}

static inline struct folio *folio_prealloc(struct mm_struct *src_mm,
                struct vm_area_struct *vma, unsigned long addr, bool need_zero)
{
        struct folio *new_folio;

        if (need_zero)
                new_folio = vma_alloc_zeroed_movable_folio(vma, addr);
        else
                new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr);

        if (!new_folio)
                return NULL;

        if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) {
                folio_put(new_folio);
                return NULL;
        }
        folio_throttle_swaprate(new_folio, GFP_KERNEL);

        return new_folio;
}

static int
copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
               pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
               unsigned long end)
{
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        struct mm_struct *src_mm = src_vma->vm_mm;
        pte_t *orig_src_pte, *orig_dst_pte;
        pte_t *src_pte, *dst_pte;
        pmd_t dummy_pmdval;
        pte_t ptent;
        spinlock_t *src_ptl, *dst_ptl;
        int progress, max_nr, ret = 0;
        int rss[NR_MM_COUNTERS];
        swp_entry_t entry = (swp_entry_t){0};
        struct folio *prealloc = NULL;
        int nr;

again:
        progress = 0;
        init_rss_vec(rss);

        /*
         * copy_pmd_range()'s prior pmd_none_or_clear_bad(src_pmd), and the
         * error handling here, assume that exclusive mmap_lock on dst and src
         * protects anon from unexpected THP transitions; with shmem and file
         * protected by mmap_lock-less collapse skipping areas with anon_vma
         * (whereas vma_needs_copy() skips areas without anon_vma).  A rework
         * can remove such assumptions later, but this is good enough for now.
         */
        dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
        if (!dst_pte) {
                ret = -ENOMEM;
                goto out;
        }

        /*
         * We already hold the exclusive mmap_lock, the copy_pte_range() and
         * retract_page_tables() are using vma->anon_vma to be exclusive, so
         * the PTE page is stable, and there is no need to get pmdval and do
         * pmd_same() check.
         */
        src_pte = pte_offset_map_rw_nolock(src_mm, src_pmd, addr, &dummy_pmdval,
                                           &src_ptl);
        if (!src_pte) {
                pte_unmap_unlock(dst_pte, dst_ptl);
                /* ret == 0 */
                goto out;
        }
        spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
        orig_src_pte = src_pte;
        orig_dst_pte = dst_pte;
        arch_enter_lazy_mmu_mode();

        do {
                nr = 1;

                /*
                 * We are holding two locks at this point - either of them
                 * could generate latencies in another task on another CPU.
                 */
                if (progress >= 32) {
                        progress = 0;
                        if (need_resched() ||
                            spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
                                break;
                }
                ptent = ptep_get(src_pte);
                if (pte_none(ptent)) {
                        progress++;
                        continue;
                }
                if (unlikely(!pte_present(ptent))) {
                        ret = copy_nonpresent_pte(dst_mm, src_mm,
                                                  dst_pte, src_pte,
                                                  dst_vma, src_vma,
                                                  addr, rss);
                        if (ret == -EIO) {
                                entry = pte_to_swp_entry(ptep_get(src_pte));
                                break;
                        } else if (ret == -EBUSY) {
                                break;
                        } else if (!ret) {
                                progress += 8;
                                continue;
                        }
                        ptent = ptep_get(src_pte);
                        VM_WARN_ON_ONCE(!pte_present(ptent));

                        /*
                         * Device exclusive entry restored, continue by copying
                         * the now present pte.
                         */
                        WARN_ON_ONCE(ret != -ENOENT);
                }
                /* copy_present_ptes() will clear `*prealloc' if consumed */
                max_nr = (end - addr) / PAGE_SIZE;
                ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte,
                                        ptent, addr, max_nr, rss, &prealloc);
                /*
                 * If we need a pre-allocated page for this pte, drop the
                 * locks, allocate, and try again.
                 * If copy failed due to hwpoison in source page, break out.
                 */
                if (unlikely(ret == -EAGAIN || ret == -EHWPOISON))
                        break;
                if (unlikely(prealloc)) {
                        /*
                         * pre-alloc page cannot be reused by next time so as
                         * to strictly follow mempolicy (e.g., alloc_page_vma()
                         * will allocate page according to address).  This
                         * could only happen if one pinned pte changed.
                         */
                        folio_put(prealloc);
                        prealloc = NULL;
                }
                nr = ret;
                progress += 8 * nr;
        } while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr,
                 addr != end);

        arch_leave_lazy_mmu_mode();
        pte_unmap_unlock(orig_src_pte, src_ptl);
        add_mm_rss_vec(dst_mm, rss);
        pte_unmap_unlock(orig_dst_pte, dst_ptl);
        cond_resched();

        if (ret == -EIO) {
                VM_WARN_ON_ONCE(!entry.val);
                if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
                        ret = -ENOMEM;
                        goto out;
                }
                entry.val = 0;
        } else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) {
                goto out;
        } else if (ret ==  -EAGAIN) {
                prealloc = folio_prealloc(src_mm, src_vma, addr, false);
                if (!prealloc)
                        return -ENOMEM;
        } else if (ret < 0) {
                VM_WARN_ON_ONCE(1);
        }

        /* We've captured and resolved the error. Reset, try again. */
        ret = 0;

        if (addr != end)
                goto again;
out:
        if (unlikely(prealloc))
                folio_put(prealloc);
        return ret;
}

static inline int
copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
               pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
               unsigned long end)
{
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        struct mm_struct *src_mm = src_vma->vm_mm;
        pmd_t *src_pmd, *dst_pmd;
        unsigned long next;

        dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
        if (!dst_pmd)
                return -ENOMEM;
        src_pmd = pmd_offset(src_pud, addr);
        do {
                next = pmd_addr_end(addr, end);
                if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
                        || pmd_devmap(*src_pmd)) {
                        int err;
                        VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
                        err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd,
                                            addr, dst_vma, src_vma);
                        if (err == -ENOMEM)
                                return -ENOMEM;
                        if (!err)
                                continue;
                        /* fall through */
                }
                if (pmd_none_or_clear_bad(src_pmd))
                        continue;
                if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
                                   addr, next))
                        return -ENOMEM;
        } while (dst_pmd++, src_pmd++, addr = next, addr != end);
        return 0;
}

static inline int
copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
               p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
               unsigned long end)
{
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        struct mm_struct *src_mm = src_vma->vm_mm;
        pud_t *src_pud, *dst_pud;
        unsigned long next;

        dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
        if (!dst_pud)
                return -ENOMEM;
        src_pud = pud_offset(src_p4d, addr);
        do {
                next = pud_addr_end(addr, end);
                if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
                        int err;

                        VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
                        err = copy_huge_pud(dst_mm, src_mm,
                                            dst_pud, src_pud, addr, src_vma);
                        if (err == -ENOMEM)
                                return -ENOMEM;
                        if (!err)
                                continue;
                        /* fall through */
                }
                if (pud_none_or_clear_bad(src_pud))
                        continue;
                if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
                                   addr, next))
                        return -ENOMEM;
        } while (dst_pud++, src_pud++, addr = next, addr != end);
        return 0;
}

static inline int
copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
               pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
               unsigned long end)
{
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        p4d_t *src_p4d, *dst_p4d;
        unsigned long next;

        dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
        if (!dst_p4d)
                return -ENOMEM;
        src_p4d = p4d_offset(src_pgd, addr);
        do {
                next = p4d_addr_end(addr, end);
                if (p4d_none_or_clear_bad(src_p4d))
                        continue;
                if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
                                   addr, next))
                        return -ENOMEM;
        } while (dst_p4d++, src_p4d++, addr = next, addr != end);
        return 0;
}

/*
 * Return true if the vma needs to copy the pgtable during this fork().  Return
 * false when we can speed up fork() by allowing lazy page faults later until
 * when the child accesses the memory range.
 */
static bool
vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
{
        /*
         * Always copy pgtables when dst_vma has uffd-wp enabled even if it's
         * file-backed (e.g. shmem). Because when uffd-wp is enabled, pgtable
         * contains uffd-wp protection information, that's something we can't
         * retrieve from page cache, and skip copying will lose those info.
         */
        if (userfaultfd_wp(dst_vma))
                return true;

        if (src_vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
                return true;

        if (src_vma->anon_vma)
                return true;

        /*
         * Don't copy ptes where a page fault will fill them correctly.  Fork
         * becomes much lighter when there are big shared or private readonly
         * mappings. The tradeoff is that copy_page_range is more efficient
         * than faulting.
         */
        return false;
}

int
copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
{
        pgd_t *src_pgd, *dst_pgd;
        unsigned long next;
        unsigned long addr = src_vma->vm_start;
        unsigned long end = src_vma->vm_end;
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        struct mm_struct *src_mm = src_vma->vm_mm;
        struct mmu_notifier_range range;
        bool is_cow;
        int ret;

        if (!vma_needs_copy(dst_vma, src_vma))
                return 0;

        if (is_vm_hugetlb_page(src_vma))
                return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);

        if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
                /*
                 * We do not free on error cases below as remove_vma
                 * gets called on error from higher level routine
                 */
                ret = track_pfn_copy(src_vma);
                if (ret)
                        return ret;
        }

        /*
         * We need to invalidate the secondary MMU mappings only when
         * there could be a permission downgrade on the ptes of the
         * parent mm. And a permission downgrade will only happen if
         * is_cow_mapping() returns true.
         */
        is_cow = is_cow_mapping(src_vma->vm_flags);

        if (is_cow) {
                mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
                                        0, src_mm, addr, end);
                mmu_notifier_invalidate_range_start(&range);
                /*
                 * Disabling preemption is not needed for the write side, as
                 * the read side doesn't spin, but goes to the mmap_lock.
                 *
                 * Use the raw variant of the seqcount_t write API to avoid
                 * lockdep complaining about preemptibility.
                 */
                vma_assert_write_locked(src_vma);
                raw_write_seqcount_begin(&src_mm->write_protect_seq);
        }

        ret = 0;
        dst_pgd = pgd_offset(dst_mm, addr);
        src_pgd = pgd_offset(src_mm, addr);
        do {
                next = pgd_addr_end(addr, end);
                if (pgd_none_or_clear_bad(src_pgd))
                        continue;
                if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
                                            addr, next))) {
                        untrack_pfn_clear(dst_vma);
                        ret = -ENOMEM;
                        break;
                }
        } while (dst_pgd++, src_pgd++, addr = next, addr != end);

        if (is_cow) {
                raw_write_seqcount_end(&src_mm->write_protect_seq);
                mmu_notifier_invalidate_range_end(&range);
        }
        return ret;
}

/* Whether we should zap all COWed (private) pages too */
static inline bool should_zap_cows(struct zap_details *details)
{
        /* By default, zap all pages */
        if (!details)
                return true;

        /* Or, we zap COWed pages only if the caller wants to */
        return details->even_cows;
}

/* Decides whether we should zap this folio with the folio pointer specified */
static inline bool should_zap_folio(struct zap_details *details,
                                    struct folio *folio)
{
        /* If we can make a decision without *folio.. */
        if (should_zap_cows(details))
                return true;

        /* Otherwise we should only zap non-anon folios */
        return !folio_test_anon(folio);
}

static inline bool zap_drop_markers(struct zap_details *details)
{
        if (!details)
                return false;

        return details->zap_flags & ZAP_FLAG_DROP_MARKER;
}

/*
 * This function makes sure that we'll replace the none pte with an uffd-wp
 * swap special pte marker when necessary. Must be with the pgtable lock held.
 */
static inline void
zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
                              unsigned long addr, pte_t *pte, int nr,
                              struct zap_details *details, pte_t pteval)
{
        /* Zap on anonymous always means dropping everything */
        if (vma_is_anonymous(vma))
                return;

        if (zap_drop_markers(details))
                return;

        for (;;) {
                /* the PFN in the PTE is irrelevant. */
                pte_install_uffd_wp_if_needed(vma, addr, pte, pteval);
                if (--nr == 0)
                        break;
                pte++;
                addr += PAGE_SIZE;
        }
}

static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,
                struct vm_area_struct *vma, struct folio *folio,
                struct page *page, pte_t *pte, pte_t ptent, unsigned int nr,
                unsigned long addr, struct zap_details *details, int *rss,
                bool *force_flush, bool *force_break)
{
        struct mm_struct *mm = tlb->mm;
        bool delay_rmap = false;

        if (!folio_test_anon(folio)) {
                ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);
                if (pte_dirty(ptent)) {
                        folio_mark_dirty(folio);
                        if (tlb_delay_rmap(tlb)) {
                                delay_rmap = true;
                                *force_flush = true;
                        }
                }
                if (pte_young(ptent) && likely(vma_has_recency(vma)))
                        folio_mark_accessed(folio);
                rss[mm_counter(folio)] -= nr;
        } else {
                /* We don't need up-to-date accessed/dirty bits. */
                clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);
                rss[MM_ANONPAGES] -= nr;
        }
        /* Checking a single PTE in a batch is sufficient. */
        arch_check_zapped_pte(vma, ptent);
        tlb_remove_tlb_entries(tlb, pte, nr, addr);
        if (unlikely(userfaultfd_pte_wp(vma, ptent)))
                zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details,
                                              ptent);

        if (!delay_rmap) {
                folio_remove_rmap_ptes(folio, page, nr, vma);

                if (unlikely(folio_mapcount(folio) < 0))
                        print_bad_pte(vma, addr, ptent, page);
        }
        if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) {
                *force_flush = true;
                *force_break = true;
        }
}

/*
 * Zap or skip at least one present PTE, trying to batch-process subsequent
 * PTEs that map consecutive pages of the same folio.
 *
 * Returns the number of processed (skipped or zapped) PTEs (at least 1).
 */
static inline int zap_present_ptes(struct mmu_gather *tlb,
                struct vm_area_struct *vma, pte_t *pte, pte_t ptent,
                unsigned int max_nr, unsigned long addr,
                struct zap_details *details, int *rss, bool *force_flush,
                bool *force_break)
{
        const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
        struct mm_struct *mm = tlb->mm;
        struct folio *folio;
        struct page *page;
        int nr;

        page = vm_normal_page(vma, addr, ptent);
        if (!page) {
                /* We don't need up-to-date accessed/dirty bits. */
                ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm);
                arch_check_zapped_pte(vma, ptent);
                tlb_remove_tlb_entry(tlb, pte, addr);
                if (userfaultfd_pte_wp(vma, ptent))
                        zap_install_uffd_wp_if_needed(vma, addr, pte, 1,
                                                      details, ptent);
                ksm_might_unmap_zero_page(mm, ptent);
                return 1;
        }

        folio = page_folio(page);
        if (unlikely(!should_zap_folio(details, folio)))
                return 1;

        /*
         * Make sure that the common "small folio" case is as fast as possible
         * by keeping the batching logic separate.
         */
        if (unlikely(folio_test_large(folio) && max_nr != 1)) {
                nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags,
                                     NULL, NULL, NULL);

                zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr,
                                       addr, details, rss, force_flush,
                                       force_break);
                return nr;
        }
        zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, 1, addr,
                               details, rss, force_flush, force_break);
        return 1;
}

static unsigned long zap_pte_range(struct mmu_gather *tlb,
                                struct vm_area_struct *vma, pmd_t *pmd,
                                unsigned long addr, unsigned long end,
                                struct zap_details *details)
{
        bool force_flush = false, force_break = false;
        struct mm_struct *mm = tlb->mm;
        int rss[NR_MM_COUNTERS];
        spinlock_t *ptl;
        pte_t *start_pte;
        pte_t *pte;
        swp_entry_t entry;
        int nr;

        tlb_change_page_size(tlb, PAGE_SIZE);
        init_rss_vec(rss);
        start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
        if (!pte)
                return addr;

        flush_tlb_batched_pending(mm);
        arch_enter_lazy_mmu_mode();
        do {
                pte_t ptent = ptep_get(pte);
                struct folio *folio;
                struct page *page;
                int max_nr;

                nr = 1;
                if (pte_none(ptent))
                        continue;

                if (need_resched())
                        break;

                if (pte_present(ptent)) {
                        max_nr = (end - addr) / PAGE_SIZE;
                        nr = zap_present_ptes(tlb, vma, pte, ptent, max_nr,
                                              addr, details, rss, &force_flush,
                                              &force_break);
                        if (unlikely(force_break)) {
                                addr += nr * PAGE_SIZE;
                                break;
                        }
                        continue;
                }

                entry = pte_to_swp_entry(ptent);
                if (is_device_private_entry(entry) ||
                    is_device_exclusive_entry(entry)) {
                        page = pfn_swap_entry_to_page(entry);
                        folio = page_folio(page);
                        if (unlikely(!should_zap_folio(details, folio)))
                                continue;
                        /*
                         * Both device private/exclusive mappings should only
                         * work with anonymous page so far, so we don't need to
                         * consider uffd-wp bit when zap. For more information,
                         * see zap_install_uffd_wp_if_needed().
                         */
                        WARN_ON_ONCE(!vma_is_anonymous(vma));
                        rss[mm_counter(folio)]--;
                        if (is_device_private_entry(entry))
                                folio_remove_rmap_pte(folio, page, vma);
                        folio_put(folio);
                } else if (!non_swap_entry(entry)) {
                        max_nr = (end - addr) / PAGE_SIZE;
                        nr = swap_pte_batch(pte, max_nr, ptent);
                        /* Genuine swap entries, hence a private anon pages */
                        if (!should_zap_cows(details))
                                continue;
                        rss[MM_SWAPENTS] -= nr;
                        free_swap_and_cache_nr(entry, nr);
                } else if (is_migration_entry(entry)) {
                        folio = pfn_swap_entry_folio(entry);
                        if (!should_zap_folio(details, folio))
                                continue;
                        rss[mm_counter(folio)]--;
                } else if (pte_marker_entry_uffd_wp(entry)) {
                        /*
                         * For anon: always drop the marker; for file: only
                         * drop the marker if explicitly requested.
                         */
                        if (!vma_is_anonymous(vma) &&
                            !zap_drop_markers(details))
                                continue;
                } else if (is_guard_swp_entry(entry)) {
                        /*
                         * Ordinary zapping should not remove guard PTE
                         * markers. Only do so if we should remove PTE markers
                         * in general.
                         */
                        if (!zap_drop_markers(details))
                                continue;
                } else if (is_hwpoison_entry(entry) ||
                           is_poisoned_swp_entry(entry)) {
                        if (!should_zap_cows(details))
                                continue;
                } else {
                        /* We should have covered all the swap entry types */
                        pr_alert("unrecognized swap entry 0x%lx\n", entry.val);
                        WARN_ON_ONCE(1);
                }
                clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm);
                zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent);
        } while (pte += nr, addr += PAGE_SIZE * nr, addr != end);

        add_mm_rss_vec(mm, rss);
        arch_leave_lazy_mmu_mode();

        /* Do the actual TLB flush before dropping ptl */
        if (force_flush) {
                tlb_flush_mmu_tlbonly(tlb);
                tlb_flush_rmaps(tlb, vma);
        }
        pte_unmap_unlock(start_pte, ptl);

        /*
         * If we forced a TLB flush (either due to running out of
         * batch buffers or because we needed to flush dirty TLB
         * entries before releasing the ptl), free the batched
         * memory too. Come back again if we didn't do everything.
         */
        if (force_flush)
                tlb_flush_mmu(tlb);

        return addr;
}

static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
                                struct vm_area_struct *vma, pud_t *pud,
                                unsigned long addr, unsigned long end,
                                struct zap_details *details)
{
        pmd_t *pmd;
        unsigned long next;

        pmd = pmd_offset(pud, addr);
        do {
                next = pmd_addr_end(addr, end);
                if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
                        if (next - addr != HPAGE_PMD_SIZE)
                                __split_huge_pmd(vma, pmd, addr, false, NULL);
                        else if (zap_huge_pmd(tlb, vma, pmd, addr)) {
                                addr = next;
                                continue;
                        }
                        /* fall through */
                } else if (details && details->single_folio &&
                           folio_test_pmd_mappable(details->single_folio) &&
                           next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) {
                        spinlock_t *ptl = pmd_lock(tlb->mm, pmd);
                        /*
                         * Take and drop THP pmd lock so that we cannot return
                         * prematurely, while zap_huge_pmd() has cleared *pmd,
                         * but not yet decremented compound_mapcount().
                         */
                        spin_unlock(ptl);
                }
                if (pmd_none(*pmd)) {
                        addr = next;
                        continue;
                }
                addr = zap_pte_range(tlb, vma, pmd, addr, next, details);
                if (addr != next)
                        pmd--;
        } while (pmd++, cond_resched(), addr != end);

        return addr;
}

static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
                                struct vm_area_struct *vma, p4d_t *p4d,
                                unsigned long addr, unsigned long end,
                                struct zap_details *details)
{
        pud_t *pud;
        unsigned long next;

        pud = pud_offset(p4d, addr);
        do {
                next = pud_addr_end(addr, end);
                if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
                        if (next - addr != HPAGE_PUD_SIZE) {
                                mmap_assert_locked(tlb->mm);
                                split_huge_pud(vma, pud, addr);
                        } else if (zap_huge_pud(tlb, vma, pud, addr))
                                goto next;
                        /* fall through */
                }
                if (pud_none_or_clear_bad(pud))
                        continue;
                next = zap_pmd_range(tlb, vma, pud, addr, next, details);
next:
                cond_resched();
        } while (pud++, addr = next, addr != end);

        return addr;
}

static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
                                struct vm_area_struct *vma, pgd_t *pgd,
                                unsigned long addr, unsigned long end,
                                struct zap_details *details)
{
        p4d_t *p4d;
        unsigned long next;

        p4d = p4d_offset(pgd, addr);
        do {
                next = p4d_addr_end(addr, end);
                if (p4d_none_or_clear_bad(p4d))
                        continue;
                next = zap_pud_range(tlb, vma, p4d, addr, next, details);
        } while (p4d++, addr = next, addr != end);

        return addr;
}

void unmap_page_range(struct mmu_gather *tlb,
                             struct vm_area_struct *vma,
                             unsigned long addr, unsigned long end,
                             struct zap_details *details)
{
        pgd_t *pgd;
        unsigned long next;

        BUG_ON(addr >= end);
        tlb_start_vma(tlb, vma);
        pgd = pgd_offset(vma->vm_mm, addr);
        do {
                next = pgd_addr_end(addr, end);
                if (pgd_none_or_clear_bad(pgd))
                        continue;
                next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
        } while (pgd++, addr = next, addr != end);
        tlb_end_vma(tlb, vma);
}


static void unmap_single_vma(struct mmu_gather *tlb,
                struct vm_area_struct *vma, unsigned long start_addr,
                unsigned long end_addr,
                struct zap_details *details, bool mm_wr_locked)
{
        unsigned long start = max(vma->vm_start, start_addr);
        unsigned long end;

        if (start >= vma->vm_end)
                return;
        end = min(vma->vm_end, end_addr);
        if (end <= vma->vm_start)
                return;

        if (vma->vm_file)
                uprobe_munmap(vma, start, end);

        if (unlikely(vma->vm_flags & VM_PFNMAP))
                untrack_pfn(vma, 0, 0, mm_wr_locked);

        if (start != end) {
                if (unlikely(is_vm_hugetlb_page(vma))) {
                        /*
                         * It is undesirable to test vma->vm_file as it
                         * should be non-null for valid hugetlb area.
                         * However, vm_file will be NULL in the error
                         * cleanup path of mmap_region. When
                         * hugetlbfs ->mmap method fails,
                         * mmap_region() nullifies vma->vm_file
                         * before calling this function to clean up.
                         * Since no pte has actually been setup, it is
                         * safe to do nothing in this case.
                         */
                        if (vma->vm_file) {
                                zap_flags_t zap_flags = details ?
                                    details->zap_flags : 0;
                                __unmap_hugepage_range(tlb, vma, start, end,
                                                             NULL, zap_flags);
                        }
                } else
                        unmap_page_range(tlb, vma, start, end, details);
        }
}

/**
 * unmap_vmas - unmap a range of memory covered by a list of vma's
 * @tlb: address of the caller's struct mmu_gather
 * @mas: the maple state
 * @vma: the starting vma
 * @start_addr: virtual address at which to start unmapping
 * @end_addr: virtual address at which to end unmapping
 * @tree_end: The maximum index to check
 * @mm_wr_locked: lock flag
 *
 * Unmap all pages in the vma list.
 *
 * Only addresses between `start' and `end' will be unmapped.
 *
 * The VMA list must be sorted in ascending virtual address order.
 *
 * unmap_vmas() assumes that the caller will flush the whole unmapped address
 * range after unmap_vmas() returns.  So the only responsibility here is to
 * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
 * drops the lock and schedules.
 */
void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas,
                struct vm_area_struct *vma, unsigned long start_addr,
                unsigned long end_addr, unsigned long tree_end,
                bool mm_wr_locked)
{
        struct mmu_notifier_range range;
        struct zap_details details = {
                .zap_flags = ZAP_FLAG_DROP_MARKER | ZAP_FLAG_UNMAP,
                /* Careful - we need to zap private pages too! */
                .even_cows = true,
        };

        mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm,
                                start_addr, end_addr);
        mmu_notifier_invalidate_range_start(&range);
        do {
                unsigned long start = start_addr;
                unsigned long end = end_addr;
                hugetlb_zap_begin(vma, &start, &end);
                unmap_single_vma(tlb, vma, start, end, &details,
                                 mm_wr_locked);
                hugetlb_zap_end(vma, &details);
                vma = mas_find(mas, tree_end - 1);
        } while (vma && likely(!xa_is_zero(vma)));
        mmu_notifier_invalidate_range_end(&range);
}

/**
 * zap_page_range_single - remove user pages in a given range
 * @vma: vm_area_struct holding the applicable pages
 * @address: starting address of pages to zap
 * @size: number of bytes to zap
 * @details: details of shared cache invalidation
 *
 * The range must fit into one VMA.
 */
void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
                unsigned long size, struct zap_details *details)
{
        const unsigned long end = address + size;
        struct mmu_notifier_range range;
        struct mmu_gather tlb;

        lru_add_drain();
        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
                                address, end);
        hugetlb_zap_begin(vma, &range.start, &range.end);
        tlb_gather_mmu(&tlb, vma->vm_mm);
        update_hiwater_rss(vma->vm_mm);
        mmu_notifier_invalidate_range_start(&range);
        /*
         * unmap 'address-end' not 'range.start-range.end' as range
         * could have been expanded for hugetlb pmd sharing.
         */
        unmap_single_vma(&tlb, vma, address, end, details, false);
        mmu_notifier_invalidate_range_end(&range);
        tlb_finish_mmu(&tlb);
        hugetlb_zap_end(vma, details);
}

/**
 * zap_vma_ptes - remove ptes mapping the vma
 * @vma: vm_area_struct holding ptes to be zapped
 * @address: starting address of pages to zap
 * @size: number of bytes to zap
 *
 * This function only unmaps ptes assigned to VM_PFNMAP vmas.
 *
 * The entire address range must be fully contained within the vma.
 *
 */
void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
                unsigned long size)
{
        if (!range_in_vma(vma, address, address + size) ||
                            !(vma->vm_flags & VM_PFNMAP))
                return;

        zap_page_range_single(vma, address, size, NULL);
}
EXPORT_SYMBOL_GPL(zap_vma_ptes);

static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
{
        pgd_t *pgd;
        p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;

        pgd = pgd_offset(mm, addr);
        p4d = p4d_alloc(mm, pgd, addr);
        if (!p4d)
                return NULL;
        pud = pud_alloc(mm, p4d, addr);
        if (!pud)
                return NULL;
        pmd = pmd_alloc(mm, pud, addr);
        if (!pmd)
                return NULL;

        VM_BUG_ON(pmd_trans_huge(*pmd));
        return pmd;
}

pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
                        spinlock_t **ptl)
{
        pmd_t *pmd = walk_to_pmd(mm, addr);

        if (!pmd)
                return NULL;
        return pte_alloc_map_lock(mm, pmd, addr, ptl);
}

static bool vm_mixed_zeropage_allowed(struct vm_area_struct *vma)
{
        VM_WARN_ON_ONCE(vma->vm_flags & VM_PFNMAP);
        /*
         * Whoever wants to forbid the zeropage after some zeropages
         * might already have been mapped has to scan the page tables and
         * bail out on any zeropages. Zeropages in COW mappings can
         * be unshared using FAULT_FLAG_UNSHARE faults.
         */
        if (mm_forbids_zeropage(vma->vm_mm))
                return false;
        /* zeropages in COW mappings are common and unproblematic. */
        if (is_cow_mapping(vma->vm_flags))
                return true;
        /* Mappings that do not allow for writable PTEs are unproblematic. */
        if (!(vma->vm_flags & (VM_WRITE | VM_MAYWRITE)))
                return true;
        /*
         * Why not allow any VMA that has vm_ops->pfn_mkwrite? GUP could
         * find the shared zeropage and longterm-pin it, which would
         * be problematic as soon as the zeropage gets replaced by a different
         * page due to vma->vm_ops->pfn_mkwrite, because what's mapped would
         * now differ to what GUP looked up. FSDAX is incompatible to
         * FOLL_LONGTERM and VM_IO is incompatible to GUP completely (see
         * check_vma_flags).
         */
        return vma->vm_ops && vma->vm_ops->pfn_mkwrite &&
               (vma_is_fsdax(vma) || vma->vm_flags & VM_IO);
}

static int validate_page_before_insert(struct vm_area_struct *vma,
                                       struct page *page)
{
        struct folio *folio = page_folio(page);

        if (!folio_ref_count(folio))
                return -EINVAL;
        if (unlikely(is_zero_folio(folio))) {
                if (!vm_mixed_zeropage_allowed(vma))
                        return -EINVAL;
                return 0;
        }
        if (folio_test_anon(folio) || folio_test_slab(folio) ||
            page_has_type(page))
                return -EINVAL;
        flush_dcache_folio(folio);
        return 0;
}

static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
                        unsigned long addr, struct page *page, pgprot_t prot)
{
        struct folio *folio = page_folio(page);
        pte_t pteval;

        if (!pte_none(ptep_get(pte)))
                return -EBUSY;
        /* Ok, finally just insert the thing.. */
        pteval = mk_pte(page, prot);
        if (unlikely(is_zero_folio(folio))) {
                pteval = pte_mkspecial(pteval);
        } else {
                folio_get(folio);
                inc_mm_counter(vma->vm_mm, mm_counter_file(folio));
                folio_add_file_rmap_pte(folio, page, vma);
        }
        set_pte_at(vma->vm_mm, addr, pte, pteval);
        return 0;
}

static int insert_page(struct vm_area_struct *vma, unsigned long addr,
                        struct page *page, pgprot_t prot)
{
        int retval;
        pte_t *pte;
        spinlock_t *ptl;

        retval = validate_page_before_insert(vma, page);
        if (retval)
                goto out;
        retval = -ENOMEM;
        pte = get_locked_pte(vma->vm_mm, addr, &ptl);
        if (!pte)
                goto out;
        retval = insert_page_into_pte_locked(vma, pte, addr, page, prot);
        pte_unmap_unlock(pte, ptl);
out:
        return retval;
}

static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
                        unsigned long addr, struct page *page, pgprot_t prot)
{
        int err;

        err = validate_page_before_insert(vma, page);
        if (err)
                return err;
        return insert_page_into_pte_locked(vma, pte, addr, page, prot);
}

/* insert_pages() amortizes the cost of spinlock operations
 * when inserting pages in a loop.
 */
static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
                        struct page **pages, unsigned long *num, pgprot_t prot)
{
        pmd_t *pmd = NULL;
        pte_t *start_pte, *pte;
        spinlock_t *pte_lock;
        struct mm_struct *const mm = vma->vm_mm;
        unsigned long curr_page_idx = 0;
        unsigned long remaining_pages_total = *num;
        unsigned long pages_to_write_in_pmd;
        int ret;
more:
        ret = -EFAULT;
        pmd = walk_to_pmd(mm, addr);
        if (!pmd)
                goto out;

        pages_to_write_in_pmd = min_t(unsigned long,
                remaining_pages_total, PTRS_PER_PTE - pte_index(addr));

        /* Allocate the PTE if necessary; takes PMD lock once only. */
        ret = -ENOMEM;
        if (pte_alloc(mm, pmd))
                goto out;

        while (pages_to_write_in_pmd) {
                int pte_idx = 0;
                const int batch_size = min_t(int, pages_to_write_in_pmd, 8);

                start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
                if (!start_pte) {
                        ret = -EFAULT;
                        goto out;
                }
                for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
                        int err = insert_page_in_batch_locked(vma, pte,
                                addr, pages[curr_page_idx], prot);
                        if (unlikely(err)) {
                                pte_unmap_unlock(start_pte, pte_lock);
                                ret = err;
                                remaining_pages_total -= pte_idx;
                                goto out;
                        }
                        addr += PAGE_SIZE;
                        ++curr_page_idx;
                }
                pte_unmap_unlock(start_pte, pte_lock);
                pages_to_write_in_pmd -= batch_size;
                remaining_pages_total -= batch_size;
        }
        if (remaining_pages_total)
                goto more;
        ret = 0;
out:
        *num = remaining_pages_total;
        return ret;
}

/**
 * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
 * @vma: user vma to map to
 * @addr: target start user address of these pages
 * @pages: source kernel pages
 * @num: in: number of pages to map. out: number of pages that were *not*
 * mapped. (0 means all pages were successfully mapped).
 *
 * Preferred over vm_insert_page() when inserting multiple pages.
 *
 * In case of error, we may have mapped a subset of the provided
 * pages. It is the caller's responsibility to account for this case.
 *
 * The same restrictions apply as in vm_insert_page().
 */
int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
                        struct page **pages, unsigned long *num)
{
        const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;

        if (addr < vma->vm_start || end_addr >= vma->vm_end)
                return -EFAULT;
        if (!(vma->vm_flags & VM_MIXEDMAP)) {
                BUG_ON(mmap_read_trylock(vma->vm_mm));
                BUG_ON(vma->vm_flags & VM_PFNMAP);
                vm_flags_set(vma, VM_MIXEDMAP);
        }
        /* Defer page refcount checking till we're about to map that page. */
        return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_insert_pages);

/**
 * vm_insert_page - insert single page into user vma
 * @vma: user vma to map to
 * @addr: target user address of this page
 * @page: source kernel page
 *
 * This allows drivers to insert individual pages they've allocated
 * into a user vma. The zeropage is supported in some VMAs,
 * see vm_mixed_zeropage_allowed().
 *
 * The page has to be a nice clean _individual_ kernel allocation.
 * If you allocate a compound page, you need to have marked it as
 * such (__GFP_COMP), or manually just split the page up yourself
 * (see split_page()).
 *
 * NOTE! Traditionally this was done with "remap_pfn_range()" which
 * took an arbitrary page protection parameter. This doesn't allow
 * that. Your vma protection will have to be set up correctly, which
 * means that if you want a shared writable mapping, you'd better
 * ask for a shared writable mapping!
 *
 * The page does not need to be reserved.
 *
 * Usually this function is called from f_op->mmap() handler
 * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
 * Caller must set VM_MIXEDMAP on vma if it wants to call this
 * function from other places, for example from page-fault handler.
 *
 * Return: %0 on success, negative error code otherwise.
 */
int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
                        struct page *page)
{
        if (addr < vma->vm_start || addr >= vma->vm_end)
                return -EFAULT;
        if (!(vma->vm_flags & VM_MIXEDMAP)) {
                BUG_ON(mmap_read_trylock(vma->vm_mm));
                BUG_ON(vma->vm_flags & VM_PFNMAP);
                vm_flags_set(vma, VM_MIXEDMAP);
        }
        return insert_page(vma, addr, page, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_insert_page);

/*
 * __vm_map_pages - maps range of kernel pages into user vma
 * @vma: user vma to map to
 * @pages: pointer to array of source kernel pages
 * @num: number of pages in page array
 * @offset: user's requested vm_pgoff
 *
 * This allows drivers to map range of kernel pages into a user vma.
 * The zeropage is supported in some VMAs, see
 * vm_mixed_zeropage_allowed().
 *
 * Return: 0 on success and error code otherwise.
 */
static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
                                unsigned long num, unsigned long offset)
{
        unsigned long count = vma_pages(vma);
        unsigned long uaddr = vma->vm_start;
        int ret, i;

        /* Fail if the user requested offset is beyond the end of the object */
        if (offset >= num)
                return -ENXIO;

        /* Fail if the user requested size exceeds available object size */
        if (count > num - offset)
                return -ENXIO;

        for (i = 0; i < count; i++) {
                ret = vm_insert_page(vma, uaddr, pages[offset + i]);
                if (ret < 0)
                        return ret;
                uaddr += PAGE_SIZE;
        }

        return 0;
}

/**
 * vm_map_pages - maps range of kernel pages starts with non zero offset
 * @vma: user vma to map to
 * @pages: pointer to array of source kernel pages
 * @num: number of pages in page array
 *
 * Maps an object consisting of @num pages, catering for the user's
 * requested vm_pgoff
 *
 * If we fail to insert any page into the vma, the function will return
 * immediately leaving any previously inserted pages present.  Callers
 * from the mmap handler may immediately return the error as their caller
 * will destroy the vma, removing any successfully inserted pages. Other
 * callers should make their own arrangements for calling unmap_region().
 *
 * Context: Process context. Called by mmap handlers.
 * Return: 0 on success and error code otherwise.
 */
int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
                                unsigned long num)
{
        return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
}
EXPORT_SYMBOL(vm_map_pages);

/**
 * vm_map_pages_zero - map range of kernel pages starts with zero offset
 * @vma: user vma to map to
 * @pages: pointer to array of source kernel pages
 * @num: number of pages in page array
 *
 * Similar to vm_map_pages(), except that it explicitly sets the offset
 * to 0. This function is intended for the drivers that did not consider
 * vm_pgoff.
 *
 * Context: Process context. Called by mmap handlers.
 * Return: 0 on success and error code otherwise.
 */
int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
                                unsigned long num)
{
        return __vm_map_pages(vma, pages, num, 0);
}
EXPORT_SYMBOL(vm_map_pages_zero);

static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
                        pfn_t pfn, pgprot_t prot, bool mkwrite)
{
        struct mm_struct *mm = vma->vm_mm;
        pte_t *pte, entry;
        spinlock_t *ptl;

        pte = get_locked_pte(mm, addr, &ptl);
        if (!pte)
                return VM_FAULT_OOM;
        entry = ptep_get(pte);
        if (!pte_none(entry)) {
                if (mkwrite) {
                        /*
                         * For read faults on private mappings the PFN passed
                         * in may not match the PFN we have mapped if the
                         * mapped PFN is a writeable COW page.  In the mkwrite
                         * case we are creating a writable PTE for a shared
                         * mapping and we expect the PFNs to match. If they
                         * don't match, we are likely racing with block
                         * allocation and mapping invalidation so just skip the
                         * update.
                         */
                        if (pte_pfn(entry) != pfn_t_to_pfn(pfn)) {
                                WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry)));
                                goto out_unlock;
                        }
                        entry = pte_mkyoung(entry);
                        entry = maybe_mkwrite(pte_mkdirty(entry), vma);
                        if (ptep_set_access_flags(vma, addr, pte, entry, 1))
                                update_mmu_cache(vma, addr, pte);
                }
                goto out_unlock;
        }

        /* Ok, finally just insert the thing.. */
        if (pfn_t_devmap(pfn))
                entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
        else
                entry = pte_mkspecial(pfn_t_pte(pfn, prot));

        if (mkwrite) {
                entry = pte_mkyoung(entry);
                entry = maybe_mkwrite(pte_mkdirty(entry), vma);
        }

        set_pte_at(mm, addr, pte, entry);
        update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */

out_unlock:
        pte_unmap_unlock(pte, ptl);
        return VM_FAULT_NOPAGE;
}

/**
 * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
 * @vma: user vma to map to
 * @addr: target user address of this page
 * @pfn: source kernel pfn
 * @pgprot: pgprot flags for the inserted page
 *
 * This is exactly like vmf_insert_pfn(), except that it allows drivers
 * to override pgprot on a per-page basis.
 *
 * This only makes sense for IO mappings, and it makes no sense for
 * COW mappings.  In general, using multiple vmas is preferable;
 * vmf_insert_pfn_prot should only be used if using multiple VMAs is
 * impractical.
 *
 * pgprot typically only differs from @vma->vm_page_prot when drivers set
 * caching- and encryption bits different than those of @vma->vm_page_prot,
 * because the caching- or encryption mode may not be known at mmap() time.
 *
 * This is ok as long as @vma->vm_page_prot is not used by the core vm
 * to set caching and encryption bits for those vmas (except for COW pages).
 * This is ensured by core vm only modifying these page table entries using
 * functions that don't touch caching- or encryption bits, using pte_modify()
 * if needed. (See for example mprotect()).
 *
 * Also when new page-table entries are created, this is only done using the
 * fault() callback, and never using the value of vma->vm_page_prot,
 * except for page-table entries that point to anonymous pages as the result
 * of COW.
 *
 * Context: Process context.  May allocate using %GFP_KERNEL.
 * Return: vm_fault_t value.
 */
vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
                        unsigned long pfn, pgprot_t pgprot)
{
        /*
         * Technically, architectures with pte_special can avoid all these
         * restrictions (same for remap_pfn_range).  However we would like
         * consistency in testing and feature parity among all, so we should
         * try to keep these invariants in place for everybody.
         */
        BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
        BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
                                                (VM_PFNMAP|VM_MIXEDMAP));
        BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
        BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));

        if (addr < vma->vm_start || addr >= vma->vm_end)
                return VM_FAULT_SIGBUS;

        if (!pfn_modify_allowed(pfn, pgprot))
                return VM_FAULT_SIGBUS;

        track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));

        return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
                        false);
}
EXPORT_SYMBOL(vmf_insert_pfn_prot);

/**
 * vmf_insert_pfn - insert single pfn into user vma
 * @vma: user vma to map to
 * @addr: target user address of this page
 * @pfn: source kernel pfn
 *
 * Similar to vm_insert_page, this allows drivers to insert individual pages
 * they've allocated into a user vma. Same comments apply.
 *
 * This function should only be called from a vm_ops->fault handler, and
 * in that case the handler should return the result of this function.
 *
 * vma cannot be a COW mapping.
 *
 * As this is called only for pages that do not currently exist, we
 * do not need to flush old virtual caches or the TLB.
 *
 * Context: Process context.  May allocate using %GFP_KERNEL.
 * Return: vm_fault_t value.
 */
vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
                        unsigned long pfn)
{
        return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
}
EXPORT_SYMBOL(vmf_insert_pfn);

static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn, bool mkwrite)
{
        if (unlikely(is_zero_pfn(pfn_t_to_pfn(pfn))) &&
            (mkwrite || !vm_mixed_zeropage_allowed(vma)))
                return false;
        /* these checks mirror the abort conditions in vm_normal_page */
        if (vma->vm_flags & VM_MIXEDMAP)
                return true;
        if (pfn_t_devmap(pfn))
                return true;
        if (pfn_t_special(pfn))
                return true;
        if (is_zero_pfn(pfn_t_to_pfn(pfn)))
                return true;
        return false;
}

static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
                unsigned long addr, pfn_t pfn, bool mkwrite)
{
        pgprot_t pgprot = vma->vm_page_prot;
        int err;

        if (!vm_mixed_ok(vma, pfn, mkwrite))
                return VM_FAULT_SIGBUS;

        if (addr < vma->vm_start || addr >= vma->vm_end)
                return VM_FAULT_SIGBUS;

        track_pfn_insert(vma, &pgprot, pfn);

        if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
                return VM_FAULT_SIGBUS;

        /*
         * If we don't have pte special, then we have to use the pfn_valid()
         * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
         * refcount the page if pfn_valid is true (hence insert_page rather
         * than insert_pfn).  If a zero_pfn were inserted into a VM_MIXEDMAP
         * without pte special, it would there be refcounted as a normal page.
         */
        if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
            !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
                struct page *page;

                /*
                 * At this point we are committed to insert_page()
                 * regardless of whether the caller specified flags that
                 * result in pfn_t_has_page() == false.
                 */
                page = pfn_to_page(pfn_t_to_pfn(pfn));
                err = insert_page(vma, addr, page, pgprot);
        } else {
                return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
        }

        if (err == -ENOMEM)
                return VM_FAULT_OOM;
        if (err < 0 && err != -EBUSY)
                return VM_FAULT_SIGBUS;

        return VM_FAULT_NOPAGE;
}

vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
                pfn_t pfn)
{
        return __vm_insert_mixed(vma, addr, pfn, false);
}
EXPORT_SYMBOL(vmf_insert_mixed);

/*
 *  If the insertion of PTE failed because someone else already added a
 *  different entry in the mean time, we treat that as success as we assume
 *  the same entry was actually inserted.
 */
vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
                unsigned long addr, pfn_t pfn)
{
        return __vm_insert_mixed(vma, addr, pfn, true);
}

/*
 * maps a range of physical memory into the requested pages. the old
 * mappings are removed. any references to nonexistent pages results
 * in null mappings (currently treated as "copy-on-access")
 */
static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
                        unsigned long addr, unsigned long end,
                        unsigned long pfn, pgprot_t prot)
{
        pte_t *pte, *mapped_pte;
        spinlock_t *ptl;
        int err = 0;

        mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
        if (!pte)
                return -ENOMEM;
        arch_enter_lazy_mmu_mode();
        do {
                BUG_ON(!pte_none(ptep_get(pte)));
                if (!pfn_modify_allowed(pfn, prot)) {
                        err = -EACCES;
                        break;
                }
                set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
                pfn++;
        } while (pte++, addr += PAGE_SIZE, addr != end);
        arch_leave_lazy_mmu_mode();
        pte_unmap_unlock(mapped_pte, ptl);
        return err;
}

static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
                        unsigned long addr, unsigned long end,
                        unsigned long pfn, pgprot_t prot)
{
        pmd_t *pmd;
        unsigned long next;
        int err;

        pfn -= addr >> PAGE_SHIFT;
        pmd = pmd_alloc(mm, pud, addr);
        if (!pmd)
                return -ENOMEM;
        VM_BUG_ON(pmd_trans_huge(*pmd));
        do {
                next = pmd_addr_end(addr, end);
                err = remap_pte_range(mm, pmd, addr, next,
                                pfn + (addr >> PAGE_SHIFT), prot);
                if (err)
                        return err;
        } while (pmd++, addr = next, addr != end);
        return 0;
}

static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
                        unsigned long addr, unsigned long end,
                        unsigned long pfn, pgprot_t prot)
{
        pud_t *pud;
        unsigned long next;
        int err;

        pfn -= addr >> PAGE_SHIFT;
        pud = pud_alloc(mm, p4d, addr);
        if (!pud)
                return -ENOMEM;
        do {
                next = pud_addr_end(addr, end);
                err = remap_pmd_range(mm, pud, addr, next,
                                pfn + (addr >> PAGE_SHIFT), prot);
                if (err)
                        return err;
        } while (pud++, addr = next, addr != end);
        return 0;
}

static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
                        unsigned long addr, unsigned long end,
                        unsigned long pfn, pgprot_t prot)
{
        p4d_t *p4d;
        unsigned long next;
        int err;

        pfn -= addr >> PAGE_SHIFT;
        p4d = p4d_alloc(mm, pgd, addr);
        if (!p4d)
                return -ENOMEM;
        do {
                next = p4d_addr_end(addr, end);
                err = remap_pud_range(mm, p4d, addr, next,
                                pfn + (addr >> PAGE_SHIFT), prot);
                if (err)
                        return err;
        } while (p4d++, addr = next, addr != end);
        return 0;
}

static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr,
                unsigned long pfn, unsigned long size, pgprot_t prot)
{
        pgd_t *pgd;
        unsigned long next;
        unsigned long end = addr + PAGE_ALIGN(size);
        struct mm_struct *mm = vma->vm_mm;
        int err;

        if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
                return -EINVAL;

        /*
         * Physically remapped pages are special. Tell the
         * rest of the world about it:
         *   VM_IO tells people not to look at these pages
         *        (accesses can have side effects).
         *   VM_PFNMAP tells the core MM that the base pages are just
         *        raw PFN mappings, and do not have a "struct page" associated
         *        with them.
         *   VM_DONTEXPAND
         *      Disable vma merging and expanding with mremap().
         *   VM_DONTDUMP
         *      Omit vma from core dump, even when VM_IO turned off.
         *
         * There's a horrible special case to handle copy-on-write
         * behaviour that some programs depend on. We mark the "original"
         * un-COW'ed pages by matching them up with "vma->vm_pgoff".
         * See vm_normal_page() for details.
         */
        if (is_cow_mapping(vma->vm_flags)) {
                if (addr != vma->vm_start || end != vma->vm_end)
                        return -EINVAL;
                vma->vm_pgoff = pfn;
        }

        vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);

        BUG_ON(addr >= end);
        pfn -= addr >> PAGE_SHIFT;
        pgd = pgd_offset(mm, addr);
        flush_cache_range(vma, addr, end);
        do {
                next = pgd_addr_end(addr, end);
                err = remap_p4d_range(mm, pgd, addr, next,
                                pfn + (addr >> PAGE_SHIFT), prot);
                if (err)
                        return err;
        } while (pgd++, addr = next, addr != end);

        return 0;
}

/*
 * Variant of remap_pfn_range that does not call track_pfn_remap.  The caller
 * must have pre-validated the caching bits of the pgprot_t.
 */
int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
                unsigned long pfn, unsigned long size, pgprot_t prot)
{
        int error = remap_pfn_range_internal(vma, addr, pfn, size, prot);

        if (!error)
                return 0;

        /*
         * A partial pfn range mapping is dangerous: it does not
         * maintain page reference counts, and callers may free
         * pages due to the error. So zap it early.
         */
        zap_page_range_single(vma, addr, size, NULL);
        return error;
}

/**
 * remap_pfn_range - remap kernel memory to userspace
 * @vma: user vma to map to
 * @addr: target page aligned user address to start at
 * @pfn: page frame number of kernel physical memory address
 * @size: size of mapping area
 * @prot: page protection flags for this mapping
 *
 * Note: this is only safe if the mm semaphore is held when called.
 *
 * Return: %0 on success, negative error code otherwise.
 */
int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
                    unsigned long pfn, unsigned long size, pgprot_t prot)
{
        int err;

        err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size));
        if (err)
                return -EINVAL;

        err = remap_pfn_range_notrack(vma, addr, pfn, size, prot);
        if (err)
                untrack_pfn(vma, pfn, PAGE_ALIGN(size), true);
        return err;
}
EXPORT_SYMBOL(remap_pfn_range);

/**
 * vm_iomap_memory - remap memory to userspace
 * @vma: user vma to map to
 * @start: start of the physical memory to be mapped
 * @len: size of area
 *
 * This is a simplified io_remap_pfn_range() for common driver use. The
 * driver just needs to give us the physical memory range to be mapped,
 * we'll figure out the rest from the vma information.
 *
 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
 * whatever write-combining details or similar.
 *
 * Return: %0 on success, negative error code otherwise.
 */
int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
{
        unsigned long vm_len, pfn, pages;

        /* Check that the physical memory area passed in looks valid */
        if (start + len < start)
                return -EINVAL;
        /*
         * You *really* shouldn't map things that aren't page-aligned,
         * but we've historically allowed it because IO memory might
         * just have smaller alignment.
         */
        len += start & ~PAGE_MASK;
        pfn = start >> PAGE_SHIFT;
        pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
        if (pfn + pages < pfn)
                return -EINVAL;

        /* We start the mapping 'vm_pgoff' pages into the area */
        if (vma->vm_pgoff > pages)
                return -EINVAL;
        pfn += vma->vm_pgoff;
        pages -= vma->vm_pgoff;

        /* Can we fit all of the mapping? */
        vm_len = vma->vm_end - vma->vm_start;
        if (vm_len >> PAGE_SHIFT > pages)
                return -EINVAL;

        /* Ok, let it rip */
        return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_iomap_memory);

static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
                                     unsigned long addr, unsigned long end,
                                     pte_fn_t fn, void *data, bool create,
                                     pgtbl_mod_mask *mask)
{
        pte_t *pte, *mapped_pte;
        int err = 0;
        spinlock_t *ptl;

        if (create) {
                mapped_pte = pte = (mm == &init_mm) ?
                        pte_alloc_kernel_track(pmd, addr, mask) :
                        pte_alloc_map_lock(mm, pmd, addr, &ptl);
                if (!pte)
                        return -ENOMEM;
        } else {
                mapped_pte = pte = (mm == &init_mm) ?
                        pte_offset_kernel(pmd, addr) :
                        pte_offset_map_lock(mm, pmd, addr, &ptl);
                if (!pte)
                        return -EINVAL;
        }

        arch_enter_lazy_mmu_mode();

        if (fn) {
                do {
                        if (create || !pte_none(ptep_get(pte))) {
                                err = fn(pte++, addr, data);
                                if (err)
                                        break;
                        }
                } while (addr += PAGE_SIZE, addr != end);
        }
        *mask |= PGTBL_PTE_MODIFIED;

        arch_leave_lazy_mmu_mode();

        if (mm != &init_mm)
                pte_unmap_unlock(mapped_pte, ptl);
        return err;
}

static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
                                     unsigned long addr, unsigned long end,
                                     pte_fn_t fn, void *data, bool create,
                                     pgtbl_mod_mask *mask)
{
        pmd_t *pmd;
        unsigned long next;
        int err = 0;

        BUG_ON(pud_leaf(*pud));

        if (create) {
                pmd = pmd_alloc_track(mm, pud, addr, mask);
                if (!pmd)
                        return -ENOMEM;
        } else {
                pmd = pmd_offset(pud, addr);
        }
        do {
                next = pmd_addr_end(addr, end);
                if (pmd_none(*pmd) && !create)
                        continue;
                if (WARN_ON_ONCE(pmd_leaf(*pmd)))
                        return -EINVAL;
                if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) {
                        if (!create)
                                continue;
                        pmd_clear_bad(pmd);
                }
                err = apply_to_pte_range(mm, pmd, addr, next,
                                         fn, data, create, mask);
                if (err)
                        break;
        } while (pmd++, addr = next, addr != end);

        return err;
}

static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
                                     unsigned long addr, unsigned long end,
                                     pte_fn_t fn, void *data, bool create,
                                     pgtbl_mod_mask *mask)
{
        pud_t *pud;
        unsigned long next;
        int err = 0;

        if (create) {
                pud = pud_alloc_track(mm, p4d, addr, mask);
                if (!pud)
                        return -ENOMEM;
        } else {
                pud = pud_offset(p4d, addr);
        }
        do {
                next = pud_addr_end(addr, end);
                if (pud_none(*pud) && !create)
                        continue;
                if (WARN_ON_ONCE(pud_leaf(*pud)))
                        return -EINVAL;
                if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) {
                        if (!create)
                                continue;
                        pud_clear_bad(pud);
                }
                err = apply_to_pmd_range(mm, pud, addr, next,
                                         fn, data, create, mask);
                if (err)
                        break;
        } while (pud++, addr = next, addr != end);

        return err;
}

static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
                                     unsigned long addr, unsigned long end,
                                     pte_fn_t fn, void *data, bool create,
                                     pgtbl_mod_mask *mask)
{
        p4d_t *p4d;
        unsigned long next;
        int err = 0;

        if (create) {
                p4d = p4d_alloc_track(mm, pgd, addr, mask);
                if (!p4d)
                        return -ENOMEM;
        } else {
                p4d = p4d_offset(pgd, addr);
        }
        do {
                next = p4d_addr_end(addr, end);
                if (p4d_none(*p4d) && !create)
                        continue;
                if (WARN_ON_ONCE(p4d_leaf(*p4d)))
                        return -EINVAL;
                if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) {
                        if (!create)
                                continue;
                        p4d_clear_bad(p4d);
                }
                err = apply_to_pud_range(mm, p4d, addr, next,
                                         fn, data, create, mask);
                if (err)
                        break;
        } while (p4d++, addr = next, addr != end);

        return err;
}

static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
                                 unsigned long size, pte_fn_t fn,
                                 void *data, bool create)
{
        pgd_t *pgd;
        unsigned long start = addr, next;
        unsigned long end = addr + size;
        pgtbl_mod_mask mask = 0;
        int err = 0;

        if (WARN_ON(addr >= end))
                return -EINVAL;

        pgd = pgd_offset(mm, addr);
        do {
                next = pgd_addr_end(addr, end);
                if (pgd_none(*pgd) && !create)
                        continue;
                if (WARN_ON_ONCE(pgd_leaf(*pgd)))
                        return -EINVAL;
                if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) {
                        if (!create)
                                continue;
                        pgd_clear_bad(pgd);
                }
                err = apply_to_p4d_range(mm, pgd, addr, next,
                                         fn, data, create, &mask);
                if (err)
                        break;
        } while (pgd++, addr = next, addr != end);

        if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
                arch_sync_kernel_mappings(start, start + size);

        return err;
}

/*
 * Scan a region of virtual memory, filling in page tables as necessary
 * and calling a provided function on each leaf page table.
 */
int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
                        unsigned long size, pte_fn_t fn, void *data)
{
        return __apply_to_page_range(mm, addr, size, fn, data, true);
}
EXPORT_SYMBOL_GPL(apply_to_page_range);

/*
 * Scan a region of virtual memory, calling a provided function on
 * each leaf page table where it exists.
 *
 * Unlike apply_to_page_range, this does _not_ fill in page tables
 * where they are absent.
 */
int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
                                 unsigned long size, pte_fn_t fn, void *data)
{
        return __apply_to_page_range(mm, addr, size, fn, data, false);
}
EXPORT_SYMBOL_GPL(apply_to_existing_page_range);

/*
 * handle_pte_fault chooses page fault handler according to an entry which was
 * read non-atomically.  Before making any commitment, on those architectures
 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
 * parts, do_swap_page must check under lock before unmapping the pte and
 * proceeding (but do_wp_page is only called after already making such a check;
 * and do_anonymous_page can safely check later on).
 */
static inline int pte_unmap_same(struct vm_fault *vmf)
{
        int same = 1;
#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
        if (sizeof(pte_t) > sizeof(unsigned long)) {
                spin_lock(vmf->ptl);
                same = pte_same(ptep_get(vmf->pte), vmf->orig_pte);
                spin_unlock(vmf->ptl);
        }
#endif
        pte_unmap(vmf->pte);
        vmf->pte = NULL;
        return same;
}

/*
 * Return:
 *        0:                copied succeeded
 *        -EHWPOISON:        copy failed due to hwpoison in source page
 *        -EAGAIN:        copied failed (some other reason)
 */
static inline int __wp_page_copy_user(struct page *dst, struct page *src,
                                      struct vm_fault *vmf)
{
        int ret;
        void *kaddr;
        void __user *uaddr;
        struct vm_area_struct *vma = vmf->vma;
        struct mm_struct *mm = vma->vm_mm;
        unsigned long addr = vmf->address;

        if (likely(src)) {
                if (copy_mc_user_highpage(dst, src, addr, vma))
                        return -EHWPOISON;
                return 0;
        }

        /*
         * If the source page was a PFN mapping, we don't have
         * a "struct page" for it. We do a best-effort copy by
         * just copying from the original user address. If that
         * fails, we just zero-fill it. Live with it.
         */
        kaddr = kmap_local_page(dst);
        pagefault_disable();
        uaddr = (void __user *)(addr & PAGE_MASK);

        /*
         * On architectures with software "accessed" bits, we would
         * take a double page fault, so mark it accessed here.
         */
        vmf->pte = NULL;
        if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) {
                pte_t entry;

                vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
                if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
                        /*
                         * Other thread has already handled the fault
                         * and update local tlb only
                         */
                        if (vmf->pte)
                                update_mmu_tlb(vma, addr, vmf->pte);
                        ret = -EAGAIN;
                        goto pte_unlock;
                }

                entry = pte_mkyoung(vmf->orig_pte);
                if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
                        update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1);
        }

        /*
         * This really shouldn't fail, because the page is there
         * in the page tables. But it might just be unreadable,
         * in which case we just give up and fill the result with
         * zeroes.
         */
        if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
                if (vmf->pte)
                        goto warn;

                /* Re-validate under PTL if the page is still mapped */
                vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
                if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
                        /* The PTE changed under us, update local tlb */
                        if (vmf->pte)
                                update_mmu_tlb(vma, addr, vmf->pte);
                        ret = -EAGAIN;
                        goto pte_unlock;
                }

                /*
                 * The same page can be mapped back since last copy attempt.
                 * Try to copy again under PTL.
                 */
                if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
                        /*
                         * Give a warn in case there can be some obscure
                         * use-case
                         */
warn:
                        WARN_ON_ONCE(1);
                        clear_page(kaddr);
                }
        }

        ret = 0;

pte_unlock:
        if (vmf->pte)
                pte_unmap_unlock(vmf->pte, vmf->ptl);
        pagefault_enable();
        kunmap_local(kaddr);
        flush_dcache_page(dst);

        return ret;
}

static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
{
        struct file *vm_file = vma->vm_file;

        if (vm_file)
                return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;

        /*
         * Special mappings (e.g. VDSO) do not have any file so fake
         * a default GFP_KERNEL for them.
         */
        return GFP_KERNEL;
}

/*
 * Notify the address space that the page is about to become writable so that
 * it can prohibit this or wait for the page to get into an appropriate state.
 *
 * We do this without the lock held, so that it can sleep if it needs to.
 */
static vm_fault_t do_page_mkwrite(struct vm_fault *vmf, struct folio *folio)
{
        vm_fault_t ret;
        unsigned int old_flags = vmf->flags;

        vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;

        if (vmf->vma->vm_file &&
            IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
                return VM_FAULT_SIGBUS;

        ret = vmf->vma->vm_ops->page_mkwrite(vmf);
        /* Restore original flags so that caller is not surprised */
        vmf->flags = old_flags;
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
                return ret;
        if (unlikely(!(ret & VM_FAULT_LOCKED))) {
                folio_lock(folio);
                if (!folio->mapping) {
                        folio_unlock(folio);
                        return 0; /* retry */
                }
                ret |= VM_FAULT_LOCKED;
        } else
                VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
        return ret;
}

/*
 * Handle dirtying of a page in shared file mapping on a write fault.
 *
 * The function expects the page to be locked and unlocks it.
 */
static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct address_space *mapping;
        struct folio *folio = page_folio(vmf->page);
        bool dirtied;
        bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;

        dirtied = folio_mark_dirty(folio);
        VM_BUG_ON_FOLIO(folio_test_anon(folio), folio);
        /*
         * Take a local copy of the address_space - folio.mapping may be zeroed
         * by truncate after folio_unlock().   The address_space itself remains
         * pinned by vma->vm_file's reference.  We rely on folio_unlock()'s
         * release semantics to prevent the compiler from undoing this copying.
         */
        mapping = folio_raw_mapping(folio);
        folio_unlock(folio);

        if (!page_mkwrite)
                file_update_time(vma->vm_file);

        /*
         * Throttle page dirtying rate down to writeback speed.
         *
         * mapping may be NULL here because some device drivers do not
         * set page.mapping but still dirty their pages
         *
         * Drop the mmap_lock before waiting on IO, if we can. The file
         * is pinning the mapping, as per above.
         */
        if ((dirtied || page_mkwrite) && mapping) {
                struct file *fpin;

                fpin = maybe_unlock_mmap_for_io(vmf, NULL);
                balance_dirty_pages_ratelimited(mapping);
                if (fpin) {
                        fput(fpin);
                        return VM_FAULT_COMPLETED;
                }
        }

        return 0;
}

/*
 * Handle write page faults for pages that can be reused in the current vma
 *
 * This can happen either due to the mapping being with the VM_SHARED flag,
 * or due to us being the last reference standing to the page. In either
 * case, all we need to do here is to mark the page as writable and update
 * any related book-keeping.
 */
static inline void wp_page_reuse(struct vm_fault *vmf, struct folio *folio)
        __releases(vmf->ptl)
{
        struct vm_area_struct *vma = vmf->vma;
        pte_t entry;

        VM_BUG_ON(!(vmf->flags & FAULT_FLAG_WRITE));
        VM_WARN_ON(is_zero_pfn(pte_pfn(vmf->orig_pte)));

        if (folio) {
                VM_BUG_ON(folio_test_anon(folio) &&
                          !PageAnonExclusive(vmf->page));
                /*
                 * Clear the folio's cpupid information as the existing
                 * information potentially belongs to a now completely
                 * unrelated process.
                 */
                folio_xchg_last_cpupid(folio, (1 << LAST_CPUPID_SHIFT) - 1);
        }

        flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
        entry = pte_mkyoung(vmf->orig_pte);
        entry = maybe_mkwrite(pte_mkdirty(entry), vma);
        if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
                update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
        pte_unmap_unlock(vmf->pte, vmf->ptl);
        count_vm_event(PGREUSE);
}

/*
 * We could add a bitflag somewhere, but for now, we know that all
 * vm_ops that have a ->map_pages have been audited and don't need
 * the mmap_lock to be held.
 */
static inline vm_fault_t vmf_can_call_fault(const struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;

        if (vma->vm_ops->map_pages || !(vmf->flags & FAULT_FLAG_VMA_LOCK))
                return 0;
        vma_end_read(vma);
        return VM_FAULT_RETRY;
}

/**
 * __vmf_anon_prepare - Prepare to handle an anonymous fault.
 * @vmf: The vm_fault descriptor passed from the fault handler.
 *
 * When preparing to insert an anonymous page into a VMA from a
 * fault handler, call this function rather than anon_vma_prepare().
 * If this vma does not already have an associated anon_vma and we are
 * only protected by the per-VMA lock, the caller must retry with the
 * mmap_lock held.  __anon_vma_prepare() will look at adjacent VMAs to
 * determine if this VMA can share its anon_vma, and that's not safe to
 * do with only the per-VMA lock held for this VMA.
 *
 * Return: 0 if fault handling can proceed.  Any other value should be
 * returned to the caller.
 */
vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        vm_fault_t ret = 0;

        if (likely(vma->anon_vma))
                return 0;
        if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
                if (!mmap_read_trylock(vma->vm_mm))
                        return VM_FAULT_RETRY;
        }
        if (__anon_vma_prepare(vma))
                ret = VM_FAULT_OOM;
        if (vmf->flags & FAULT_FLAG_VMA_LOCK)
                mmap_read_unlock(vma->vm_mm);
        return ret;
}

/*
 * Handle the case of a page which we actually need to copy to a new page,
 * either due to COW or unsharing.
 *
 * Called with mmap_lock locked and the old page referenced, but
 * without the ptl held.
 *
 * High level logic flow:
 *
 * - Allocate a page, copy the content of the old page to the new one.
 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
 * - Take the PTL. If the pte changed, bail out and release the allocated page
 * - If the pte is still the way we remember it, update the page table and all
 *   relevant references. This includes dropping the reference the page-table
 *   held to the old page, as well as updating the rmap.
 * - In any case, unlock the PTL and drop the reference we took to the old page.
 */
static vm_fault_t wp_page_copy(struct vm_fault *vmf)
{
        const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
        struct vm_area_struct *vma = vmf->vma;
        struct mm_struct *mm = vma->vm_mm;
        struct folio *old_folio = NULL;
        struct folio *new_folio = NULL;
        pte_t entry;
        int page_copied = 0;
        struct mmu_notifier_range range;
        vm_fault_t ret;
        bool pfn_is_zero;

        delayacct_wpcopy_start();

        if (vmf->page)
                old_folio = page_folio(vmf->page);
        ret = vmf_anon_prepare(vmf);
        if (unlikely(ret))
                goto out;

        pfn_is_zero = is_zero_pfn(pte_pfn(vmf->orig_pte));
        new_folio = folio_prealloc(mm, vma, vmf->address, pfn_is_zero);
        if (!new_folio)
                goto oom;

        if (!pfn_is_zero) {
                int err;

                err = __wp_page_copy_user(&new_folio->page, vmf->page, vmf);
                if (err) {
                        /*
                         * COW failed, if the fault was solved by other,
                         * it's fine. If not, userspace would re-fault on
                         * the same address and we will handle the fault
                         * from the second attempt.
                         * The -EHWPOISON case will not be retried.
                         */
                        folio_put(new_folio);
                        if (old_folio)
                                folio_put(old_folio);

                        delayacct_wpcopy_end();
                        return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
                }
                kmsan_copy_page_meta(&new_folio->page, vmf->page);
        }

        __folio_mark_uptodate(new_folio);

        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
                                vmf->address & PAGE_MASK,
                                (vmf->address & PAGE_MASK) + PAGE_SIZE);
        mmu_notifier_invalidate_range_start(&range);

        /*
         * Re-check the pte - we dropped the lock
         */
        vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
        if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
                if (old_folio) {
                        if (!folio_test_anon(old_folio)) {
                                dec_mm_counter(mm, mm_counter_file(old_folio));
                                inc_mm_counter(mm, MM_ANONPAGES);
                        }
                } else {
                        ksm_might_unmap_zero_page(mm, vmf->orig_pte);
                        inc_mm_counter(mm, MM_ANONPAGES);
                }
                flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
                entry = mk_pte(&new_folio->page, vma->vm_page_prot);
                entry = pte_sw_mkyoung(entry);
                if (unlikely(unshare)) {
                        if (pte_soft_dirty(vmf->orig_pte))
                                entry = pte_mksoft_dirty(entry);
                        if (pte_uffd_wp(vmf->orig_pte))
                                entry = pte_mkuffd_wp(entry);
                } else {
                        entry = maybe_mkwrite(pte_mkdirty(entry), vma);
                }

                /*
                 * Clear the pte entry and flush it first, before updating the
                 * pte with the new entry, to keep TLBs on different CPUs in
                 * sync. This code used to set the new PTE then flush TLBs, but
                 * that left a window where the new PTE could be loaded into
                 * some TLBs while the old PTE remains in others.
                 */
                ptep_clear_flush(vma, vmf->address, vmf->pte);
                folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE);
                folio_add_lru_vma(new_folio, vma);
                BUG_ON(unshare && pte_write(entry));
                set_pte_at(mm, vmf->address, vmf->pte, entry);
                update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
                if (old_folio) {
                        /*
                         * Only after switching the pte to the new page may
                         * we remove the mapcount here. Otherwise another
                         * process may come and find the rmap count decremented
                         * before the pte is switched to the new page, and
                         * "reuse" the old page writing into it while our pte
                         * here still points into it and can be read by other
                         * threads.
                         *
                         * The critical issue is to order this
                         * folio_remove_rmap_pte() with the ptp_clear_flush
                         * above. Those stores are ordered by (if nothing else,)
                         * the barrier present in the atomic_add_negative
                         * in folio_remove_rmap_pte();
                         *
                         * Then the TLB flush in ptep_clear_flush ensures that
                         * no process can access the old page before the
                         * decremented mapcount is visible. And the old page
                         * cannot be reused until after the decremented
                         * mapcount is visible. So transitively, TLBs to
                         * old page will be flushed before it can be reused.
                         */
                        folio_remove_rmap_pte(old_folio, vmf->page, vma);
                }

                /* Free the old page.. */
                new_folio = old_folio;
                page_copied = 1;
                pte_unmap_unlock(vmf->pte, vmf->ptl);
        } else if (vmf->pte) {
                update_mmu_tlb(vma, vmf->address, vmf->pte);
                pte_unmap_unlock(vmf->pte, vmf->ptl);
        }

        mmu_notifier_invalidate_range_end(&range);

        if (new_folio)
                folio_put(new_folio);
        if (old_folio) {
                if (page_copied)
                        free_swap_cache(old_folio);
                folio_put(old_folio);
        }

        delayacct_wpcopy_end();
        return 0;
oom:
        ret = VM_FAULT_OOM;
out:
        if (old_folio)
                folio_put(old_folio);

        delayacct_wpcopy_end();
        return ret;
}

/**
 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
 *                          writeable once the page is prepared
 *
 * @vmf: structure describing the fault
 * @folio: the folio of vmf->page
 *
 * This function handles all that is needed to finish a write page fault in a
 * shared mapping due to PTE being read-only once the mapped page is prepared.
 * It handles locking of PTE and modifying it.
 *
 * The function expects the page to be locked or other protection against
 * concurrent faults / writeback (such as DAX radix tree locks).
 *
 * Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before
 * we acquired PTE lock.
 */
static vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf, struct folio *folio)
{
        WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
        vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
                                       &vmf->ptl);
        if (!vmf->pte)
                return VM_FAULT_NOPAGE;
        /*
         * We might have raced with another page fault while we released the
         * pte_offset_map_lock.
         */
        if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) {
                update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
                pte_unmap_unlock(vmf->pte, vmf->ptl);
                return VM_FAULT_NOPAGE;
        }
        wp_page_reuse(vmf, folio);
        return 0;
}

/*
 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
 * mapping
 */
static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;

        if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
                vm_fault_t ret;

                pte_unmap_unlock(vmf->pte, vmf->ptl);
                ret = vmf_can_call_fault(vmf);
                if (ret)
                        return ret;

                vmf->flags |= FAULT_FLAG_MKWRITE;
                ret = vma->vm_ops->pfn_mkwrite(vmf);
                if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
                        return ret;
                return finish_mkwrite_fault(vmf, NULL);
        }
        wp_page_reuse(vmf, NULL);
        return 0;
}

static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio)
        __releases(vmf->ptl)
{
        struct vm_area_struct *vma = vmf->vma;
        vm_fault_t ret = 0;

        folio_get(folio);

        if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
                vm_fault_t tmp;

                pte_unmap_unlock(vmf->pte, vmf->ptl);
                tmp = vmf_can_call_fault(vmf);
                if (tmp) {
                        folio_put(folio);
                        return tmp;
                }

                tmp = do_page_mkwrite(vmf, folio);
                if (unlikely(!tmp || (tmp &
                                      (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
                        folio_put(folio);
                        return tmp;
                }
                tmp = finish_mkwrite_fault(vmf, folio);
                if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
                        folio_unlock(folio);
                        folio_put(folio);
                        return tmp;
                }
        } else {
                wp_page_reuse(vmf, folio);
                folio_lock(folio);
        }
        ret |= fault_dirty_shared_page(vmf);
        folio_put(folio);

        return ret;
}

static bool wp_can_reuse_anon_folio(struct folio *folio,
                                    struct vm_area_struct *vma)
{
        /*
         * We could currently only reuse a subpage of a large folio if no
         * other subpages of the large folios are still mapped. However,
         * let's just consistently not reuse subpages even if we could
         * reuse in that scenario, and give back a large folio a bit
         * sooner.
         */
        if (folio_test_large(folio))
                return false;

        /*
         * We have to verify under folio lock: these early checks are
         * just an optimization to avoid locking the folio and freeing
         * the swapcache if there is little hope that we can reuse.
         *
         * KSM doesn't necessarily raise the folio refcount.
         */
        if (folio_test_ksm(folio) || folio_ref_count(folio) > 3)
                return false;
        if (!folio_test_lru(folio))
                /*
                 * We cannot easily detect+handle references from
                 * remote LRU caches or references to LRU folios.
                 */
                lru_add_drain();
        if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio))
                return false;
        if (!folio_trylock(folio))
                return false;
        if (folio_test_swapcache(folio))
                folio_free_swap(folio);
        if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) {
                folio_unlock(folio);
                return false;
        }
        /*
         * Ok, we've got the only folio reference from our mapping
         * and the folio is locked, it's dark out, and we're wearing
         * sunglasses. Hit it.
         */
        folio_move_anon_rmap(folio, vma);
        folio_unlock(folio);
        return true;
}

/*
 * This routine handles present pages, when
 * * users try to write to a shared page (FAULT_FLAG_WRITE)
 * * GUP wants to take a R/O pin on a possibly shared anonymous page
 *   (FAULT_FLAG_UNSHARE)
 *
 * It is done by copying the page to a new address and decrementing the
 * shared-page counter for the old page.
 *
 * Note that this routine assumes that the protection checks have been
 * done by the caller (the low-level page fault routine in most cases).
 * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've
 * done any necessary COW.
 *
 * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even
 * though the page will change only once the write actually happens. This
 * avoids a few races, and potentially makes it more efficient.
 *
 * We enter with non-exclusive mmap_lock (to exclude vma changes,
 * but allow concurrent faults), with pte both mapped and locked.
 * We return with mmap_lock still held, but pte unmapped and unlocked.
 */
static vm_fault_t do_wp_page(struct vm_fault *vmf)
        __releases(vmf->ptl)
{
        const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio = NULL;
        pte_t pte;

        if (likely(!unshare)) {
                if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) {
                        if (!userfaultfd_wp_async(vma)) {
                                pte_unmap_unlock(vmf->pte, vmf->ptl);
                                return handle_userfault(vmf, VM_UFFD_WP);
                        }

                        /*
                         * Nothing needed (cache flush, TLB invalidations,
                         * etc.) because we're only removing the uffd-wp bit,
                         * which is completely invisible to the user.
                         */
                        pte = pte_clear_uffd_wp(ptep_get(vmf->pte));

                        set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
                        /*
                         * Update this to be prepared for following up CoW
                         * handling
                         */
                        vmf->orig_pte = pte;
                }

                /*
                 * Userfaultfd write-protect can defer flushes. Ensure the TLB
                 * is flushed in this case before copying.
                 */
                if (unlikely(userfaultfd_wp(vmf->vma) &&
                             mm_tlb_flush_pending(vmf->vma->vm_mm)))
                        flush_tlb_page(vmf->vma, vmf->address);
        }

        vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);

        if (vmf->page)
                folio = page_folio(vmf->page);

        /*
         * Shared mapping: we are guaranteed to have VM_WRITE and
         * FAULT_FLAG_WRITE set at this point.
         */
        if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
                /*
                 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
                 * VM_PFNMAP VMA.
                 *
                 * We should not cow pages in a shared writeable mapping.
                 * Just mark the pages writable and/or call ops->pfn_mkwrite.
                 */
                if (!vmf->page)
                        return wp_pfn_shared(vmf);
                return wp_page_shared(vmf, folio);
        }

        /*
         * Private mapping: create an exclusive anonymous page copy if reuse
         * is impossible. We might miss VM_WRITE for FOLL_FORCE handling.
         *
         * If we encounter a page that is marked exclusive, we must reuse
         * the page without further checks.
         */
        if (folio && folio_test_anon(folio) &&
            (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) {
                if (!PageAnonExclusive(vmf->page))
                        SetPageAnonExclusive(vmf->page);
                if (unlikely(unshare)) {
                        pte_unmap_unlock(vmf->pte, vmf->ptl);
                        return 0;
                }
                wp_page_reuse(vmf, folio);
                return 0;
        }
        /*
         * Ok, we need to copy. Oh, well..
         */
        if (folio)
                folio_get(folio);

        pte_unmap_unlock(vmf->pte, vmf->ptl);
#ifdef CONFIG_KSM
        if (folio && folio_test_ksm(folio))
                count_vm_event(COW_KSM);
#endif
        return wp_page_copy(vmf);
}

static void unmap_mapping_range_vma(struct vm_area_struct *vma,
                unsigned long start_addr, unsigned long end_addr,
                struct zap_details *details)
{
        zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
}

static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
                                            pgoff_t first_index,
                                            pgoff_t last_index,
                                            struct zap_details *details)
{
        struct vm_area_struct *vma;
        pgoff_t vba, vea, zba, zea;

        vma_interval_tree_foreach(vma, root, first_index, last_index) {
                vba = vma->vm_pgoff;
                vea = vba + vma_pages(vma) - 1;
                zba = max(first_index, vba);
                zea = min(last_index, vea);

                unmap_mapping_range_vma(vma,
                        ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
                        ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
                                details);
        }
}

/**
 * unmap_mapping_folio() - Unmap single folio from processes.
 * @folio: The locked folio to be unmapped.
 *
 * Unmap this folio from any userspace process which still has it mmaped.
 * Typically, for efficiency, the range of nearby pages has already been
 * unmapped by unmap_mapping_pages() or unmap_mapping_range().  But once
 * truncation or invalidation holds the lock on a folio, it may find that
 * the page has been remapped again: and then uses unmap_mapping_folio()
 * to unmap it finally.
 */
void unmap_mapping_folio(struct folio *folio)
{
        struct address_space *mapping = folio->mapping;
        struct zap_details details = { };
        pgoff_t        first_index;
        pgoff_t        last_index;

        VM_BUG_ON(!folio_test_locked(folio));

        first_index = folio->index;
        last_index = folio_next_index(folio) - 1;

        details.even_cows = false;
        details.single_folio = folio;
        details.zap_flags = ZAP_FLAG_DROP_MARKER;

        i_mmap_lock_read(mapping);
        if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
                unmap_mapping_range_tree(&mapping->i_mmap, first_index,
                                         last_index, &details);
        i_mmap_unlock_read(mapping);
}

/**
 * unmap_mapping_pages() - Unmap pages from processes.
 * @mapping: The address space containing pages to be unmapped.
 * @start: Index of first page to be unmapped.
 * @nr: Number of pages to be unmapped.  0 to unmap to end of file.
 * @even_cows: Whether to unmap even private COWed pages.
 *
 * Unmap the pages in this address space from any userspace process which
 * has them mmaped.  Generally, you want to remove COWed pages as well when
 * a file is being truncated, but not when invalidating pages from the page
 * cache.
 */
void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
                pgoff_t nr, bool even_cows)
{
        struct zap_details details = { };
        pgoff_t        first_index = start;
        pgoff_t        last_index = start + nr - 1;

        details.even_cows = even_cows;
        if (last_index < first_index)
                last_index = ULONG_MAX;

        i_mmap_lock_read(mapping);
        if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
                unmap_mapping_range_tree(&mapping->i_mmap, first_index,
                                         last_index, &details);
        i_mmap_unlock_read(mapping);
}
EXPORT_SYMBOL_GPL(unmap_mapping_pages);

/**
 * unmap_mapping_range - unmap the portion of all mmaps in the specified
 * address_space corresponding to the specified byte range in the underlying
 * file.
 *
 * @mapping: the address space containing mmaps to be unmapped.
 * @holebegin: byte in first page to unmap, relative to the start of
 * the underlying file.  This will be rounded down to a PAGE_SIZE
 * boundary.  Note that this is different from truncate_pagecache(), which
 * must keep the partial page.  In contrast, we must get rid of
 * partial pages.
 * @holelen: size of prospective hole in bytes.  This will be rounded
 * up to a PAGE_SIZE boundary.  A holelen of zero truncates to the
 * end of the file.
 * @even_cows: 1 when truncating a file, unmap even private COWed pages;
 * but 0 when invalidating pagecache, don't throw away private data.
 */
void unmap_mapping_range(struct address_space *mapping,
                loff_t const holebegin, loff_t const holelen, int even_cows)
{
        pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT;
        pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT;

        /* Check for overflow. */
        if (sizeof(holelen) > sizeof(hlen)) {
                long long holeend =
                        (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
                if (holeend & ~(long long)ULONG_MAX)
                        hlen = ULONG_MAX - hba + 1;
        }

        unmap_mapping_pages(mapping, hba, hlen, even_cows);
}
EXPORT_SYMBOL(unmap_mapping_range);

/*
 * Restore a potential device exclusive pte to a working pte entry
 */
static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
{
        struct folio *folio = page_folio(vmf->page);
        struct vm_area_struct *vma = vmf->vma;
        struct mmu_notifier_range range;
        vm_fault_t ret;

        /*
         * We need a reference to lock the folio because we don't hold
         * the PTL so a racing thread can remove the device-exclusive
         * entry and unmap it. If the folio is free the entry must
         * have been removed already. If it happens to have already
         * been re-allocated after being freed all we do is lock and
         * unlock it.
         */
        if (!folio_try_get(folio))
                return 0;

        ret = folio_lock_or_retry(folio, vmf);
        if (ret) {
                folio_put(folio);
                return ret;
        }
        mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
                                vma->vm_mm, vmf->address & PAGE_MASK,
                                (vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
        mmu_notifier_invalidate_range_start(&range);

        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
                                &vmf->ptl);
        if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
                restore_exclusive_pte(vma, vmf->page, vmf->address, vmf->pte);

        if (vmf->pte)
                pte_unmap_unlock(vmf->pte, vmf->ptl);
        folio_unlock(folio);
        folio_put(folio);

        mmu_notifier_invalidate_range_end(&range);
        return 0;
}

static inline bool should_try_to_free_swap(struct folio *folio,
                                           struct vm_area_struct *vma,
                                           unsigned int fault_flags)
{
        if (!folio_test_swapcache(folio))
                return false;
        if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) ||
            folio_test_mlocked(folio))
                return true;
        /*
         * If we want to map a page that's in the swapcache writable, we
         * have to detect via the refcount if we're really the exclusive
         * user. Try freeing the swapcache to get rid of the swapcache
         * reference only in case it's likely that we'll be the exlusive user.
         */
        return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) &&
                folio_ref_count(folio) == (1 + folio_nr_pages(folio));
}

static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
{
        vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
                                       vmf->address, &vmf->ptl);
        if (!vmf->pte)
                return 0;
        /*
         * Be careful so that we will only recover a special uffd-wp pte into a
         * none pte.  Otherwise it means the pte could have changed, so retry.
         *
         * This should also cover the case where e.g. the pte changed
         * quickly from a PTE_MARKER_UFFD_WP into PTE_MARKER_POISONED.
         * So is_pte_marker() check is not enough to safely drop the pte.
         */
        if (pte_same(vmf->orig_pte, ptep_get(vmf->pte)))
                pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
        pte_unmap_unlock(vmf->pte, vmf->ptl);
        return 0;
}

static vm_fault_t do_pte_missing(struct vm_fault *vmf)
{
        if (vma_is_anonymous(vmf->vma))
                return do_anonymous_page(vmf);
        else
                return do_fault(vmf);
}

/*
 * This is actually a page-missing access, but with uffd-wp special pte
 * installed.  It means this pte was wr-protected before being unmapped.
 */
static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
{
        /*
         * Just in case there're leftover special ptes even after the region
         * got unregistered - we can simply clear them.
         */
        if (unlikely(!userfaultfd_wp(vmf->vma)))
                return pte_marker_clear(vmf);

        return do_pte_missing(vmf);
}

static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
{
        swp_entry_t entry = pte_to_swp_entry(vmf->orig_pte);
        unsigned long marker = pte_marker_get(entry);

        /*
         * PTE markers should never be empty.  If anything weird happened,
         * the best thing to do is to kill the process along with its mm.
         */
        if (WARN_ON_ONCE(!marker))
                return VM_FAULT_SIGBUS;

        /* Higher priority than uffd-wp when data corrupted */
        if (marker & PTE_MARKER_POISONED)
                return VM_FAULT_HWPOISON;

        /* Hitting a guard page is always a fatal condition. */
        if (marker & PTE_MARKER_GUARD)
                return VM_FAULT_SIGSEGV;

        if (pte_marker_entry_uffd_wp(entry))
                return pte_marker_handle_uffd_wp(vmf);

        /* This is an unknown pte marker */
        return VM_FAULT_SIGBUS;
}

static struct folio *__alloc_swap_folio(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio;
        swp_entry_t entry;

        folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, vmf->address);
        if (!folio)
                return NULL;

        entry = pte_to_swp_entry(vmf->orig_pte);
        if (mem_cgroup_swapin_charge_folio(folio, vma->vm_mm,
                                           GFP_KERNEL, entry)) {
                folio_put(folio);
                return NULL;
        }

        return folio;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
{
        struct swap_info_struct *si = swp_swap_info(entry);
        pgoff_t offset = swp_offset(entry);
        int i;

        /*
         * While allocating a large folio and doing swap_read_folio, which is
         * the case the being faulted pte doesn't have swapcache. We need to
         * ensure all PTEs have no cache as well, otherwise, we might go to
         * swap devices while the content is in swapcache.
         */
        for (i = 0; i < max_nr; i++) {
                if ((si->swap_map[offset + i] & SWAP_HAS_CACHE))
                        return i;
        }

        return i;
}

/*
 * Check if the PTEs within a range are contiguous swap entries
 * and have consistent swapcache, zeromap.
 */
static bool can_swapin_thp(struct vm_fault *vmf, pte_t *ptep, int nr_pages)
{
        unsigned long addr;
        swp_entry_t entry;
        int idx;
        pte_t pte;

        addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE);
        idx = (vmf->address - addr) / PAGE_SIZE;
        pte = ptep_get(ptep);

        if (!pte_same(pte, pte_move_swp_offset(vmf->orig_pte, -idx)))
                return false;
        entry = pte_to_swp_entry(pte);
        if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages)
                return false;

        /*
         * swap_read_folio() can't handle the case a large folio is hybridly
         * from different backends. And they are likely corner cases. Similar
         * things might be added once zswap support large folios.
         */
        if (unlikely(swap_zeromap_batch(entry, nr_pages, NULL) != nr_pages))
                return false;
        if (unlikely(non_swapcache_batch(entry, nr_pages) != nr_pages))
                return false;

        return true;
}

static inline unsigned long thp_swap_suitable_orders(pgoff_t swp_offset,
                                                     unsigned long addr,
                                                     unsigned long orders)
{
        int order, nr;

        order = highest_order(orders);

        /*
         * To swap in a THP with nr pages, we require that its first swap_offset
         * is aligned with that number, as it was when the THP was swapped out.
         * This helps filter out most invalid entries.
         */
        while (orders) {
                nr = 1 << order;
                if ((addr >> PAGE_SHIFT) % nr == swp_offset % nr)
                        break;
                order = next_order(&orders, order);
        }

        return orders;
}

static struct folio *alloc_swap_folio(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        unsigned long orders;
        struct folio *folio;
        unsigned long addr;
        swp_entry_t entry;
        spinlock_t *ptl;
        pte_t *pte;
        gfp_t gfp;
        int order;

        /*
         * If uffd is active for the vma we need per-page fault fidelity to
         * maintain the uffd semantics.
         */
        if (unlikely(userfaultfd_armed(vma)))
                goto fallback;

        /*
         * A large swapped out folio could be partially or fully in zswap. We
         * lack handling for such cases, so fallback to swapping in order-0
         * folio.
         */
        if (!zswap_never_enabled())
                goto fallback;

        entry = pte_to_swp_entry(vmf->orig_pte);
        /*
         * Get a list of all the (large) orders below PMD_ORDER that are enabled
         * and suitable for swapping THP.
         */
        orders = thp_vma_allowable_orders(vma, vma->vm_flags,
                        TVA_IN_PF | TVA_ENFORCE_SYSFS, BIT(PMD_ORDER) - 1);
        orders = thp_vma_suitable_orders(vma, vmf->address, orders);
        orders = thp_swap_suitable_orders(swp_offset(entry),
                                          vmf->address, orders);

        if (!orders)
                goto fallback;

        pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
                                  vmf->address & PMD_MASK, &ptl);
        if (unlikely(!pte))
                goto fallback;

        /*
         * For do_swap_page, find the highest order where the aligned range is
         * completely swap entries with contiguous swap offsets.
         */
        order = highest_order(orders);
        while (orders) {
                addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
                if (can_swapin_thp(vmf, pte + pte_index(addr), 1 << order))
                        break;
                order = next_order(&orders, order);
        }

        pte_unmap_unlock(pte, ptl);

        /* Try allocating the highest of the remaining orders. */
        gfp = vma_thp_gfp_mask(vma);
        while (orders) {
                addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
                folio = vma_alloc_folio(gfp, order, vma, addr);
                if (folio) {
                        if (!mem_cgroup_swapin_charge_folio(folio, vma->vm_mm,
                                                            gfp, entry))
                                return folio;
                        folio_put(folio);
                }
                order = next_order(&orders, order);
        }

fallback:
        return __alloc_swap_folio(vmf);
}
#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
static struct folio *alloc_swap_folio(struct vm_fault *vmf)
{
        return __alloc_swap_folio(vmf);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

static DECLARE_WAIT_QUEUE_HEAD(swapcache_wq);

/*
 * We enter with non-exclusive mmap_lock (to exclude vma changes,
 * but allow concurrent faults), and pte mapped but not yet locked.
 * We return with pte unmapped and unlocked.
 *
 * We return with the mmap_lock locked or unlocked in the same cases
 * as does filemap_fault().
 */
vm_fault_t do_swap_page(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *swapcache, *folio = NULL;
        DECLARE_WAITQUEUE(wait, current);
        struct page *page;
        struct swap_info_struct *si = NULL;
        rmap_t rmap_flags = RMAP_NONE;
        bool need_clear_cache = false;
        bool exclusive = false;
        swp_entry_t entry;
        pte_t pte;
        vm_fault_t ret = 0;
        void *shadow = NULL;
        int nr_pages;
        unsigned long page_idx;
        unsigned long address;
        pte_t *ptep;

        if (!pte_unmap_same(vmf))
                goto out;

        entry = pte_to_swp_entry(vmf->orig_pte);
        if (unlikely(non_swap_entry(entry))) {
                if (is_migration_entry(entry)) {
                        migration_entry_wait(vma->vm_mm, vmf->pmd,
                                             vmf->address);
                } else if (is_device_exclusive_entry(entry)) {
                        vmf->page = pfn_swap_entry_to_page(entry);
                        ret = remove_device_exclusive_entry(vmf);
                } else if (is_device_private_entry(entry)) {
                        if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
                                /*
                                 * migrate_to_ram is not yet ready to operate
                                 * under VMA lock.
                                 */
                                vma_end_read(vma);
                                ret = VM_FAULT_RETRY;
                                goto out;
                        }

                        vmf->page = pfn_swap_entry_to_page(entry);
                        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                        vmf->address, &vmf->ptl);
                        if (unlikely(!vmf->pte ||
                                     !pte_same(ptep_get(vmf->pte),
                                                        vmf->orig_pte)))
                                goto unlock;

                        /*
                         * Get a page reference while we know the page can't be
                         * freed.
                         */
                        get_page(vmf->page);
                        pte_unmap_unlock(vmf->pte, vmf->ptl);
                        ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
                        put_page(vmf->page);
                } else if (is_hwpoison_entry(entry)) {
                        ret = VM_FAULT_HWPOISON;
                } else if (is_pte_marker_entry(entry)) {
                        ret = handle_pte_marker(vmf);
                } else {
                        print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
                        ret = VM_FAULT_SIGBUS;
                }
                goto out;
        }

        /* Prevent swapoff from happening to us. */
        si = get_swap_device(entry);
        if (unlikely(!si))
                goto out;

        folio = swap_cache_get_folio(entry, vma, vmf->address);
        if (folio)
                page = folio_file_page(folio, swp_offset(entry));
        swapcache = folio;

        if (!folio) {
                if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
                    __swap_count(entry) == 1) {
                        /* skip swapcache */
                        folio = alloc_swap_folio(vmf);
                        if (folio) {
                                __folio_set_locked(folio);
                                __folio_set_swapbacked(folio);

                                nr_pages = folio_nr_pages(folio);
                                if (folio_test_large(folio))
                                        entry.val = ALIGN_DOWN(entry.val, nr_pages);
                                /*
                                 * Prevent parallel swapin from proceeding with
                                 * the cache flag. Otherwise, another thread
                                 * may finish swapin first, free the entry, and
                                 * swapout reusing the same entry. It's
                                 * undetectable as pte_same() returns true due
                                 * to entry reuse.
                                 */
                                if (swapcache_prepare(entry, nr_pages)) {
                                        /*
                                         * Relax a bit to prevent rapid
                                         * repeated page faults.
                                         */
                                        add_wait_queue(&swapcache_wq, &wait);
                                        schedule_timeout_uninterruptible(1);
                                        remove_wait_queue(&swapcache_wq, &wait);
                                        goto out_page;
                                }
                                need_clear_cache = true;

                                mem_cgroup_swapin_uncharge_swap(entry, nr_pages);

                                shadow = get_shadow_from_swap_cache(entry);
                                if (shadow)
                                        workingset_refault(folio, shadow);

                                folio_add_lru(folio);

                                /* To provide entry to swap_read_folio() */
                                folio->swap = entry;
                                swap_read_folio(folio, NULL);
                                folio->private = NULL;
                        }
                } else {
                        folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
                                                vmf);
                        swapcache = folio;
                }

                if (!folio) {
                        /*
                         * Back out if somebody else faulted in this pte
                         * while we released the pte lock.
                         */
                        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                        vmf->address, &vmf->ptl);
                        if (likely(vmf->pte &&
                                   pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
                                ret = VM_FAULT_OOM;
                        goto unlock;
                }

                /* Had to read the page from swap area: Major fault */
                ret = VM_FAULT_MAJOR;
                count_vm_event(PGMAJFAULT);
                count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
                page = folio_file_page(folio, swp_offset(entry));
        } else if (PageHWPoison(page)) {
                /*
                 * hwpoisoned dirty swapcache pages are kept for killing
                 * owner processes (which may be unknown at hwpoison time)
                 */
                ret = VM_FAULT_HWPOISON;
                goto out_release;
        }

        ret |= folio_lock_or_retry(folio, vmf);
        if (ret & VM_FAULT_RETRY)
                goto out_release;

        if (swapcache) {
                /*
                 * Make sure folio_free_swap() or swapoff did not release the
                 * swapcache from under us.  The page pin, and pte_same test
                 * below, are not enough to exclude that.  Even if it is still
                 * swapcache, we need to check that the page's swap has not
                 * changed.
                 */
                if (unlikely(!folio_test_swapcache(folio) ||
                             page_swap_entry(page).val != entry.val))
                        goto out_page;

                /*
                 * KSM sometimes has to copy on read faults, for example, if
                 * page->index of !PageKSM() pages would be nonlinear inside the
                 * anon VMA -- PageKSM() is lost on actual swapout.
                 */
                folio = ksm_might_need_to_copy(folio, vma, vmf->address);
                if (unlikely(!folio)) {
                        ret = VM_FAULT_OOM;
                        folio = swapcache;
                        goto out_page;
                } else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
                        ret = VM_FAULT_HWPOISON;
                        folio = swapcache;
                        goto out_page;
                }
                if (folio != swapcache)
                        page = folio_page(folio, 0);

                /*
                 * If we want to map a page that's in the swapcache writable, we
                 * have to detect via the refcount if we're really the exclusive
                 * owner. Try removing the extra reference from the local LRU
                 * caches if required.
                 */
                if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
                    !folio_test_ksm(folio) && !folio_test_lru(folio))
                        lru_add_drain();
        }

        folio_throttle_swaprate(folio, GFP_KERNEL);

        /*
         * Back out if somebody else already faulted in this pte.
         */
        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
                        &vmf->ptl);
        if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
                goto out_nomap;

        if (unlikely(!folio_test_uptodate(folio))) {
                ret = VM_FAULT_SIGBUS;
                goto out_nomap;
        }

        /* allocated large folios for SWP_SYNCHRONOUS_IO */
        if (folio_test_large(folio) && !folio_test_swapcache(folio)) {
                unsigned long nr = folio_nr_pages(folio);
                unsigned long folio_start = ALIGN_DOWN(vmf->address, nr * PAGE_SIZE);
                unsigned long idx = (vmf->address - folio_start) / PAGE_SIZE;
                pte_t *folio_ptep = vmf->pte - idx;
                pte_t folio_pte = ptep_get(folio_ptep);

                if (!pte_same(folio_pte, pte_move_swp_offset(vmf->orig_pte, -idx)) ||
                    swap_pte_batch(folio_ptep, nr, folio_pte) != nr)
                        goto out_nomap;

                page_idx = idx;
                address = folio_start;
                ptep = folio_ptep;
                goto check_folio;
        }

        nr_pages = 1;
        page_idx = 0;
        address = vmf->address;
        ptep = vmf->pte;
        if (folio_test_large(folio) && folio_test_swapcache(folio)) {
                int nr = folio_nr_pages(folio);
                unsigned long idx = folio_page_idx(folio, page);
                unsigned long folio_start = address - idx * PAGE_SIZE;
                unsigned long folio_end = folio_start + nr * PAGE_SIZE;
                pte_t *folio_ptep;
                pte_t folio_pte;

                if (unlikely(folio_start < max(address & PMD_MASK, vma->vm_start)))
                        goto check_folio;
                if (unlikely(folio_end > pmd_addr_end(address, vma->vm_end)))
                        goto check_folio;

                folio_ptep = vmf->pte - idx;
                folio_pte = ptep_get(folio_ptep);
                if (!pte_same(folio_pte, pte_move_swp_offset(vmf->orig_pte, -idx)) ||
                    swap_pte_batch(folio_ptep, nr, folio_pte) != nr)
                        goto check_folio;

                page_idx = idx;
                address = folio_start;
                ptep = folio_ptep;
                nr_pages = nr;
                entry = folio->swap;
                page = &folio->page;
        }

check_folio:
        /*
         * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte
         * must never point at an anonymous page in the swapcache that is
         * PG_anon_exclusive. Sanity check that this holds and especially, that
         * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity
         * check after taking the PT lock and making sure that nobody
         * concurrently faulted in this page and set PG_anon_exclusive.
         */
        BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio));
        BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));

        /*
         * Check under PT lock (to protect against concurrent fork() sharing
         * the swap entry concurrently) for certainly exclusive pages.
         */
        if (!folio_test_ksm(folio)) {
                exclusive = pte_swp_exclusive(vmf->orig_pte);
                if (folio != swapcache) {
                        /*
                         * We have a fresh page that is not exposed to the
                         * swapcache -> certainly exclusive.
                         */
                        exclusive = true;
                } else if (exclusive && folio_test_writeback(folio) &&
                          data_race(si->flags & SWP_STABLE_WRITES)) {
                        /*
                         * This is tricky: not all swap backends support
                         * concurrent page modifications while under writeback.
                         *
                         * So if we stumble over such a page in the swapcache
                         * we must not set the page exclusive, otherwise we can
                         * map it writable without further checks and modify it
                         * while still under writeback.
                         *
                         * For these problematic swap backends, simply drop the
                         * exclusive marker: this is perfectly fine as we start
                         * writeback only if we fully unmapped the page and
                         * there are no unexpected references on the page after
                         * unmapping succeeded. After fully unmapped, no
                         * further GUP references (FOLL_GET and FOLL_PIN) can
                         * appear, so dropping the exclusive marker and mapping
                         * it only R/O is fine.
                         */
                        exclusive = false;
                }
        }

        /*
         * Some architectures may have to restore extra metadata to the page
         * when reading from swap. This metadata may be indexed by swap entry
         * so this must be called before swap_free().
         */
        arch_swap_restore(folio_swap(entry, folio), folio);

        /*
         * Remove the swap entry and conditionally try to free up the swapcache.
         * We're already holding a reference on the page but haven't mapped it
         * yet.
         */
        swap_free_nr(entry, nr_pages);
        if (should_try_to_free_swap(folio, vma, vmf->flags))
                folio_free_swap(folio);

        add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages);
        add_mm_counter(vma->vm_mm, MM_SWAPENTS, -nr_pages);
        pte = mk_pte(page, vma->vm_page_prot);
        if (pte_swp_soft_dirty(vmf->orig_pte))
                pte = pte_mksoft_dirty(pte);
        if (pte_swp_uffd_wp(vmf->orig_pte))
                pte = pte_mkuffd_wp(pte);

        /*
         * Same logic as in do_wp_page(); however, optimize for pages that are
         * certainly not shared either because we just allocated them without
         * exposing them to the swapcache or because the swap entry indicates
         * exclusivity.
         */
        if (!folio_test_ksm(folio) &&
            (exclusive || folio_ref_count(folio) == 1)) {
                if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) &&
                    !pte_needs_soft_dirty_wp(vma, pte)) {
                        pte = pte_mkwrite(pte, vma);
                        if (vmf->flags & FAULT_FLAG_WRITE) {
                                pte = pte_mkdirty(pte);
                                vmf->flags &= ~FAULT_FLAG_WRITE;
                        }
                }
                rmap_flags |= RMAP_EXCLUSIVE;
        }
        folio_ref_add(folio, nr_pages - 1);
        flush_icache_pages(vma, page, nr_pages);
        vmf->orig_pte = pte_advance_pfn(pte, page_idx);

        /* ksm created a completely new copy */
        if (unlikely(folio != swapcache && swapcache)) {
                folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE);
                folio_add_lru_vma(folio, vma);
        } else if (!folio_test_anon(folio)) {
                /*
                 * We currently only expect small !anon folios which are either
                 * fully exclusive or fully shared, or new allocated large
                 * folios which are fully exclusive. If we ever get large
                 * folios within swapcache here, we have to be careful.
                 */
                VM_WARN_ON_ONCE(folio_test_large(folio) && folio_test_swapcache(folio));
                VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
                folio_add_new_anon_rmap(folio, vma, address, rmap_flags);
        } else {
                folio_add_anon_rmap_ptes(folio, page, nr_pages, vma, address,
                                        rmap_flags);
        }

        VM_BUG_ON(!folio_test_anon(folio) ||
                        (pte_write(pte) && !PageAnonExclusive(page)));
        set_ptes(vma->vm_mm, address, ptep, pte, nr_pages);
        arch_do_swap_page_nr(vma->vm_mm, vma, address,
                        pte, pte, nr_pages);

        folio_unlock(folio);
        if (folio != swapcache && swapcache) {
                /*
                 * Hold the lock to avoid the swap entry to be reused
                 * until we take the PT lock for the pte_same() check
                 * (to avoid false positives from pte_same). For
                 * further safety release the lock after the swap_free
                 * so that the swap count won't change under a
                 * parallel locked swapcache.
                 */
                folio_unlock(swapcache);
                folio_put(swapcache);
        }

        if (vmf->flags & FAULT_FLAG_WRITE) {
                ret |= do_wp_page(vmf);
                if (ret & VM_FAULT_ERROR)
                        ret &= VM_FAULT_ERROR;
                goto out;
        }

        /* No need to invalidate - it was non-present before */
        update_mmu_cache_range(vmf, vma, address, ptep, nr_pages);
unlock:
        if (vmf->pte)
                pte_unmap_unlock(vmf->pte, vmf->ptl);
out:
        /* Clear the swap cache pin for direct swapin after PTL unlock */
        if (need_clear_cache) {
                swapcache_clear(si, entry, nr_pages);
                if (waitqueue_active(&swapcache_wq))
                        wake_up(&swapcache_wq);
        }
        if (si)
                put_swap_device(si);
        return ret;
out_nomap:
        if (vmf->pte)
                pte_unmap_unlock(vmf->pte, vmf->ptl);
out_page:
        folio_unlock(folio);
out_release:
        folio_put(folio);
        if (folio != swapcache && swapcache) {
                folio_unlock(swapcache);
                folio_put(swapcache);
        }
        if (need_clear_cache) {
                swapcache_clear(si, entry, nr_pages);
                if (waitqueue_active(&swapcache_wq))
                        wake_up(&swapcache_wq);
        }
        if (si)
                put_swap_device(si);
        return ret;
}

static bool pte_range_none(pte_t *pte, int nr_pages)
{
        int i;

        for (i = 0; i < nr_pages; i++) {
                if (!pte_none(ptep_get_lockless(pte + i)))
                        return false;
        }

        return true;
}

static struct folio *alloc_anon_folio(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
        unsigned long orders;
        struct folio *folio;
        unsigned long addr;
        pte_t *pte;
        gfp_t gfp;
        int order;

        /*
         * If uffd is active for the vma we need per-page fault fidelity to
         * maintain the uffd semantics.
         */
        if (unlikely(userfaultfd_armed(vma)))
                goto fallback;

        /*
         * Get a list of all the (large) orders below PMD_ORDER that are enabled
         * for this vma. Then filter out the orders that can't be allocated over
         * the faulting address and still be fully contained in the vma.
         */
        orders = thp_vma_allowable_orders(vma, vma->vm_flags,
                        TVA_IN_PF | TVA_ENFORCE_SYSFS, BIT(PMD_ORDER) - 1);
        orders = thp_vma_suitable_orders(vma, vmf->address, orders);

        if (!orders)
                goto fallback;

        pte = pte_offset_map(vmf->pmd, vmf->address & PMD_MASK);
        if (!pte)
                return ERR_PTR(-EAGAIN);

        /*
         * Find the highest order where the aligned range is completely
         * pte_none(). Note that all remaining orders will be completely
         * pte_none().
         */
        order = highest_order(orders);
        while (orders) {
                addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
                if (pte_range_none(pte + pte_index(addr), 1 << order))
                        break;
                order = next_order(&orders, order);
        }

        pte_unmap(pte);

        if (!orders)
                goto fallback;

        /* Try allocating the highest of the remaining orders. */
        gfp = vma_thp_gfp_mask(vma);
        while (orders) {
                addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
                folio = vma_alloc_folio(gfp, order, vma, addr);
                if (folio) {
                        if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) {
                                count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
                                folio_put(folio);
                                goto next;
                        }
                        folio_throttle_swaprate(folio, gfp);
                        /*
                         * When a folio is not zeroed during allocation
                         * (__GFP_ZERO not used), folio_zero_user() is used
                         * to make sure that the page corresponding to the
                         * faulting address will be hot in the cache after
                         * zeroing.
                         */
                        if (!alloc_zeroed())
                                folio_zero_user(folio, vmf->address);
                        return folio;
                }
next:
                count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK);
                order = next_order(&orders, order);
        }

fallback:
#endif
        return folio_prealloc(vma->vm_mm, vma, vmf->address, true);
}

/*
 * We enter with non-exclusive mmap_lock (to exclude vma changes,
 * but allow concurrent faults), and pte mapped but not yet locked.
 * We return with mmap_lock still held, but pte unmapped and unlocked.
 */
static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        unsigned long addr = vmf->address;
        struct folio *folio;
        vm_fault_t ret = 0;
        int nr_pages = 1;
        pte_t entry;

        /* File mapping without ->vm_ops ? */
        if (vma->vm_flags & VM_SHARED)
                return VM_FAULT_SIGBUS;

        /*
         * Use pte_alloc() instead of pte_alloc_map(), so that OOM can
         * be distinguished from a transient failure of pte_offset_map().
         */
        if (pte_alloc(vma->vm_mm, vmf->pmd))
                return VM_FAULT_OOM;

        /* Use the zero-page for reads */
        if (!(vmf->flags & FAULT_FLAG_WRITE) &&
                        !mm_forbids_zeropage(vma->vm_mm)) {
                entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
                                                vma->vm_page_prot));
                vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                vmf->address, &vmf->ptl);
                if (!vmf->pte)
                        goto unlock;
                if (vmf_pte_changed(vmf)) {
                        update_mmu_tlb(vma, vmf->address, vmf->pte);
                        goto unlock;
                }
                ret = check_stable_address_space(vma->vm_mm);
                if (ret)
                        goto unlock;
                /* Deliver the page fault to userland, check inside PT lock */
                if (userfaultfd_missing(vma)) {
                        pte_unmap_unlock(vmf->pte, vmf->ptl);
                        return handle_userfault(vmf, VM_UFFD_MISSING);
                }
                goto setpte;
        }

        /* Allocate our own private page. */
        ret = vmf_anon_prepare(vmf);
        if (ret)
                return ret;
        /* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */
        folio = alloc_anon_folio(vmf);
        if (IS_ERR(folio))
                return 0;
        if (!folio)
                goto oom;

        nr_pages = folio_nr_pages(folio);
        addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE);

        /*
         * The memory barrier inside __folio_mark_uptodate makes sure that
         * preceding stores to the page contents become visible before
         * the set_pte_at() write.
         */
        __folio_mark_uptodate(folio);

        entry = mk_pte(&folio->page, vma->vm_page_prot);
        entry = pte_sw_mkyoung(entry);
        if (vma->vm_flags & VM_WRITE)
                entry = pte_mkwrite(pte_mkdirty(entry), vma);

        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl);
        if (!vmf->pte)
                goto release;
        if (nr_pages == 1 && vmf_pte_changed(vmf)) {
                update_mmu_tlb(vma, addr, vmf->pte);
                goto release;
        } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
                update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages);
                goto release;
        }

        ret = check_stable_address_space(vma->vm_mm);
        if (ret)
                goto release;

        /* Deliver the page fault to userland, check inside PT lock */
        if (userfaultfd_missing(vma)) {
                pte_unmap_unlock(vmf->pte, vmf->ptl);
                folio_put(folio);
                return handle_userfault(vmf, VM_UFFD_MISSING);
        }

        folio_ref_add(folio, nr_pages - 1);
        add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages);
        count_mthp_stat(folio_order(folio), MTHP_STAT_ANON_FAULT_ALLOC);
        folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
        folio_add_lru_vma(folio, vma);
setpte:
        if (vmf_orig_pte_uffd_wp(vmf))
                entry = pte_mkuffd_wp(entry);
        set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr_pages);

        /* No need to invalidate - it was non-present before */
        update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr_pages);
unlock:
        if (vmf->pte)
                pte_unmap_unlock(vmf->pte, vmf->ptl);
        return ret;
release:
        folio_put(folio);
        goto unlock;
oom:
        return VM_FAULT_OOM;
}

/*
 * The mmap_lock must have been held on entry, and may have been
 * released depending on flags and vma->vm_ops->fault() return value.
 * See filemap_fault() and __lock_page_retry().
 */
static vm_fault_t __do_fault(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio;
        vm_fault_t ret;

        /*
         * Preallocate pte before we take page_lock because this might lead to
         * deadlocks for memcg reclaim which waits for pages under writeback:
         *                                lock_page(A)
         *                                SetPageWriteback(A)
         *                                unlock_page(A)
         * lock_page(B)
         *                                lock_page(B)
         * pte_alloc_one
         *   shrink_folio_list
         *     wait_on_page_writeback(A)
         *                                SetPageWriteback(B)
         *                                unlock_page(B)
         *                                # flush A, B to clear the writeback
         */
        if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
                vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
                if (!vmf->prealloc_pte)
                        return VM_FAULT_OOM;
        }

        ret = vma->vm_ops->fault(vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
                            VM_FAULT_DONE_COW)))
                return ret;

        folio = page_folio(vmf->page);
        if (unlikely(PageHWPoison(vmf->page))) {
                vm_fault_t poisonret = VM_FAULT_HWPOISON;
                if (ret & VM_FAULT_LOCKED) {
                        if (page_mapped(vmf->page))
                                unmap_mapping_folio(folio);
                        /* Retry if a clean folio was removed from the cache. */
                        if (mapping_evict_folio(folio->mapping, folio))
                                poisonret = VM_FAULT_NOPAGE;
                        folio_unlock(folio);
                }
                folio_put(folio);
                vmf->page = NULL;
                return poisonret;
        }

        if (unlikely(!(ret & VM_FAULT_LOCKED)))
                folio_lock(folio);
        else
                VM_BUG_ON_PAGE(!folio_test_locked(folio), vmf->page);

        return ret;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static void deposit_prealloc_pte(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;

        pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
        /*
         * We are going to consume the prealloc table,
         * count that as nr_ptes.
         */
        mm_inc_nr_ptes(vma->vm_mm);
        vmf->prealloc_pte = NULL;
}

vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
{
        struct folio *folio = page_folio(page);
        struct vm_area_struct *vma = vmf->vma;
        bool write = vmf->flags & FAULT_FLAG_WRITE;
        unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
        pmd_t entry;
        vm_fault_t ret = VM_FAULT_FALLBACK;

        /*
         * It is too late to allocate a small folio, we already have a large
         * folio in the pagecache: especially s390 KVM cannot tolerate any
         * PMD mappings, but PTE-mapped THP are fine. So let's simply refuse any
         * PMD mappings if THPs are disabled.
         */
        if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags))
                return ret;

        if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER))
                return ret;

        if (folio_order(folio) != HPAGE_PMD_ORDER)
                return ret;
        page = &folio->page;

        /*
         * Just backoff if any subpage of a THP is corrupted otherwise
         * the corrupted page may mapped by PMD silently to escape the
         * check.  This kind of THP just can be PTE mapped.  Access to
         * the corrupted subpage should trigger SIGBUS as expected.
         */
        if (unlikely(folio_test_has_hwpoisoned(folio)))
                return ret;

        /*
         * Archs like ppc64 need additional space to store information
         * related to pte entry. Use the preallocated table for that.
         */
        if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
                vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
                if (!vmf->prealloc_pte)
                        return VM_FAULT_OOM;
        }

        vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
        if (unlikely(!pmd_none(*vmf->pmd)))
                goto out;

        flush_icache_pages(vma, page, HPAGE_PMD_NR);

        entry = mk_huge_pmd(page, vma->vm_page_prot);
        if (write)
                entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);

        add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR);
        folio_add_file_rmap_pmd(folio, page, vma);

        /*
         * deposit and withdraw with pmd lock held
         */
        if (arch_needs_pgtable_deposit())
                deposit_prealloc_pte(vmf);

        set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);

        update_mmu_cache_pmd(vma, haddr, vmf->pmd);

        /* fault is handled */
        ret = 0;
        count_vm_event(THP_FILE_MAPPED);
out:
        spin_unlock(vmf->ptl);
        return ret;
}
#else
vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
{
        return VM_FAULT_FALLBACK;
}
#endif

/**
 * set_pte_range - Set a range of PTEs to point to pages in a folio.
 * @vmf: Fault decription.
 * @folio: The folio that contains @page.
 * @page: The first page to create a PTE for.
 * @nr: The number of PTEs to create.
 * @addr: The first address to create a PTE for.
 */
void set_pte_range(struct vm_fault *vmf, struct folio *folio,
                struct page *page, unsigned int nr, unsigned long addr)
{
        struct vm_area_struct *vma = vmf->vma;
        bool write = vmf->flags & FAULT_FLAG_WRITE;
        bool prefault = !in_range(vmf->address, addr, nr * PAGE_SIZE);
        pte_t entry;

        flush_icache_pages(vma, page, nr);
        entry = mk_pte(page, vma->vm_page_prot);

        if (prefault && arch_wants_old_prefaulted_pte())
                entry = pte_mkold(entry);
        else
                entry = pte_sw_mkyoung(entry);

        if (write)
                entry = maybe_mkwrite(pte_mkdirty(entry), vma);
        if (unlikely(vmf_orig_pte_uffd_wp(vmf)))
                entry = pte_mkuffd_wp(entry);
        /* copy-on-write page */
        if (write && !(vma->vm_flags & VM_SHARED)) {
                VM_BUG_ON_FOLIO(nr != 1, folio);
                folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
                folio_add_lru_vma(folio, vma);
        } else {
                folio_add_file_rmap_ptes(folio, page, nr, vma);
        }
        set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr);

        /* no need to invalidate: a not-present page won't be cached */
        update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr);
}

static bool vmf_pte_changed(struct vm_fault *vmf)
{
        if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)
                return !pte_same(ptep_get(vmf->pte), vmf->orig_pte);

        return !pte_none(ptep_get(vmf->pte));
}

/**
 * finish_fault - finish page fault once we have prepared the page to fault
 *
 * @vmf: structure describing the fault
 *
 * This function handles all that is needed to finish a page fault once the
 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
 * given page, adds reverse page mapping, handles memcg charges and LRU
 * addition.
 *
 * The function expects the page to be locked and on success it consumes a
 * reference of a page being mapped (for the PTE which maps it).
 *
 * Return: %0 on success, %VM_FAULT_ code in case of error.
 */
vm_fault_t finish_fault(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct page *page;
        struct folio *folio;
        vm_fault_t ret;
        bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) &&
                      !(vma->vm_flags & VM_SHARED);
        int type, nr_pages;
        unsigned long addr = vmf->address;

        /* Did we COW the page? */
        if (is_cow)
                page = vmf->cow_page;
        else
                page = vmf->page;

        /*
         * check even for read faults because we might have lost our CoWed
         * page
         */
        if (!(vma->vm_flags & VM_SHARED)) {
                ret = check_stable_address_space(vma->vm_mm);
                if (ret)
                        return ret;
        }

        if (pmd_none(*vmf->pmd)) {
                if (PageTransCompound(page)) {
                        ret = do_set_pmd(vmf, page);
                        if (ret != VM_FAULT_FALLBACK)
                                return ret;
                }

                if (vmf->prealloc_pte)
                        pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte);
                else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd)))
                        return VM_FAULT_OOM;
        }

        folio = page_folio(page);
        nr_pages = folio_nr_pages(folio);

        /*
         * Using per-page fault to maintain the uffd semantics, and same
         * approach also applies to non-anonymous-shmem faults to avoid
         * inflating the RSS of the process.
         */
        if (!vma_is_anon_shmem(vma) || unlikely(userfaultfd_armed(vma))) {
                nr_pages = 1;
        } else if (nr_pages > 1) {
                pgoff_t idx = folio_page_idx(folio, page);
                /* The page offset of vmf->address within the VMA. */
                pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
                /* The index of the entry in the pagetable for fault page. */
                pgoff_t pte_off = pte_index(vmf->address);

                /*
                 * Fallback to per-page fault in case the folio size in page
                 * cache beyond the VMA limits and PMD pagetable limits.
                 */
                if (unlikely(vma_off < idx ||
                            vma_off + (nr_pages - idx) > vma_pages(vma) ||
                            pte_off < idx ||
                            pte_off + (nr_pages - idx)  > PTRS_PER_PTE)) {
                        nr_pages = 1;
                } else {
                        /* Now we can set mappings for the whole large folio. */
                        addr = vmf->address - idx * PAGE_SIZE;
                        page = &folio->page;
                }
        }

        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                       addr, &vmf->ptl);
        if (!vmf->pte)
                return VM_FAULT_NOPAGE;

        /* Re-check under ptl */
        if (nr_pages == 1 && unlikely(vmf_pte_changed(vmf))) {
                update_mmu_tlb(vma, addr, vmf->pte);
                ret = VM_FAULT_NOPAGE;
                goto unlock;
        } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
                update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages);
                ret = VM_FAULT_NOPAGE;
                goto unlock;
        }

        folio_ref_add(folio, nr_pages - 1);
        set_pte_range(vmf, folio, page, nr_pages, addr);
        type = is_cow ? MM_ANONPAGES : mm_counter_file(folio);
        add_mm_counter(vma->vm_mm, type, nr_pages);
        ret = 0;

unlock:
        pte_unmap_unlock(vmf->pte, vmf->ptl);
        return ret;
}

static unsigned long fault_around_pages __read_mostly =
        65536 >> PAGE_SHIFT;

#ifdef CONFIG_DEBUG_FS
static int fault_around_bytes_get(void *data, u64 *val)
{
        *val = fault_around_pages << PAGE_SHIFT;
        return 0;
}

/*
 * fault_around_bytes must be rounded down to the nearest page order as it's
 * what do_fault_around() expects to see.
 */
static int fault_around_bytes_set(void *data, u64 val)
{
        if (val / PAGE_SIZE > PTRS_PER_PTE)
                return -EINVAL;

        /*
         * The minimum value is 1 page, however this results in no fault-around
         * at all. See should_fault_around().
         */
        val = max(val, PAGE_SIZE);
        fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT;

        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
                fault_around_bytes_get, fault_around_bytes_set, "%llu\n");

static int __init fault_around_debugfs(void)
{
        debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
                                   &fault_around_bytes_fops);
        return 0;
}
late_initcall(fault_around_debugfs);
#endif

/*
 * do_fault_around() tries to map few pages around the fault address. The hope
 * is that the pages will be needed soon and this will lower the number of
 * faults to handle.
 *
 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
 * not ready to be mapped: not up-to-date, locked, etc.
 *
 * This function doesn't cross VMA or page table boundaries, in order to call
 * map_pages() and acquire a PTE lock only once.
 *
 * fault_around_pages defines how many pages we'll try to map.
 * do_fault_around() expects it to be set to a power of two less than or equal
 * to PTRS_PER_PTE.
 *
 * The virtual address of the area that we map is naturally aligned to
 * fault_around_pages * PAGE_SIZE rounded down to the machine page size
 * (and therefore to page order).  This way it's easier to guarantee
 * that we don't cross page table boundaries.
 */
static vm_fault_t do_fault_around(struct vm_fault *vmf)
{
        pgoff_t nr_pages = READ_ONCE(fault_around_pages);
        pgoff_t pte_off = pte_index(vmf->address);
        /* The page offset of vmf->address within the VMA. */
        pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
        pgoff_t from_pte, to_pte;
        vm_fault_t ret;

        /* The PTE offset of the start address, clamped to the VMA. */
        from_pte = max(ALIGN_DOWN(pte_off, nr_pages),
                       pte_off - min(pte_off, vma_off));

        /* The PTE offset of the end address, clamped to the VMA and PTE. */
        to_pte = min3(from_pte + nr_pages, (pgoff_t)PTRS_PER_PTE,
                      pte_off + vma_pages(vmf->vma) - vma_off) - 1;

        if (pmd_none(*vmf->pmd)) {
                vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
                if (!vmf->prealloc_pte)
                        return VM_FAULT_OOM;
        }

        rcu_read_lock();
        ret = vmf->vma->vm_ops->map_pages(vmf,
                        vmf->pgoff + from_pte - pte_off,
                        vmf->pgoff + to_pte - pte_off);
        rcu_read_unlock();

        return ret;
}

/* Return true if we should do read fault-around, false otherwise */
static inline bool should_fault_around(struct vm_fault *vmf)
{
        /* No ->map_pages?  No way to fault around... */
        if (!vmf->vma->vm_ops->map_pages)
                return false;

        if (uffd_disable_fault_around(vmf->vma))
                return false;

        /* A single page implies no faulting 'around' at all. */
        return fault_around_pages > 1;
}

static vm_fault_t do_read_fault(struct vm_fault *vmf)
{
        vm_fault_t ret = 0;
        struct folio *folio;

        /*
         * Let's call ->map_pages() first and use ->fault() as fallback
         * if page by the offset is not ready to be mapped (cold cache or
         * something).
         */
        if (should_fault_around(vmf)) {
                ret = do_fault_around(vmf);
                if (ret)
                        return ret;
        }

        ret = vmf_can_call_fault(vmf);
        if (ret)
                return ret;

        ret = __do_fault(vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
                return ret;

        ret |= finish_fault(vmf);
        folio = page_folio(vmf->page);
        folio_unlock(folio);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
                folio_put(folio);
        return ret;
}

static vm_fault_t do_cow_fault(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio;
        vm_fault_t ret;

        ret = vmf_can_call_fault(vmf);
        if (!ret)
                ret = vmf_anon_prepare(vmf);
        if (ret)
                return ret;

        folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false);
        if (!folio)
                return VM_FAULT_OOM;

        vmf->cow_page = &folio->page;

        ret = __do_fault(vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
                goto uncharge_out;
        if (ret & VM_FAULT_DONE_COW)
                return ret;

        if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) {
                ret = VM_FAULT_HWPOISON;
                goto unlock;
        }
        __folio_mark_uptodate(folio);

        ret |= finish_fault(vmf);
unlock:
        unlock_page(vmf->page);
        put_page(vmf->page);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
                goto uncharge_out;
        return ret;
uncharge_out:
        folio_put(folio);
        return ret;
}

static vm_fault_t do_shared_fault(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        vm_fault_t ret, tmp;
        struct folio *folio;

        ret = vmf_can_call_fault(vmf);
        if (ret)
                return ret;

        ret = __do_fault(vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
                return ret;

        folio = page_folio(vmf->page);

        /*
         * Check if the backing address space wants to know that the page is
         * about to become writable
         */
        if (vma->vm_ops->page_mkwrite) {
                folio_unlock(folio);
                tmp = do_page_mkwrite(vmf, folio);
                if (unlikely(!tmp ||
                                (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
                        folio_put(folio);
                        return tmp;
                }
        }

        ret |= finish_fault(vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
                                        VM_FAULT_RETRY))) {
                folio_unlock(folio);
                folio_put(folio);
                return ret;
        }

        ret |= fault_dirty_shared_page(vmf);
        return ret;
}

/*
 * We enter with non-exclusive mmap_lock (to exclude vma changes,
 * but allow concurrent faults).
 * The mmap_lock may have been released depending on flags and our
 * return value.  See filemap_fault() and __folio_lock_or_retry().
 * If mmap_lock is released, vma may become invalid (for example
 * by other thread calling munmap()).
 */
static vm_fault_t do_fault(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct mm_struct *vm_mm = vma->vm_mm;
        vm_fault_t ret;

        /*
         * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
         */
        if (!vma->vm_ops->fault) {
                vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
                                               vmf->address, &vmf->ptl);
                if (unlikely(!vmf->pte))
                        ret = VM_FAULT_SIGBUS;
                else {
                        /*
                         * Make sure this is not a temporary clearing of pte
                         * by holding ptl and checking again. A R/M/W update
                         * of pte involves: take ptl, clearing the pte so that
                         * we don't have concurrent modification by hardware
                         * followed by an update.
                         */
                        if (unlikely(pte_none(ptep_get(vmf->pte))))
                                ret = VM_FAULT_SIGBUS;
                        else
                                ret = VM_FAULT_NOPAGE;

                        pte_unmap_unlock(vmf->pte, vmf->ptl);
                }
        } else if (!(vmf->flags & FAULT_FLAG_WRITE))
                ret = do_read_fault(vmf);
        else if (!(vma->vm_flags & VM_SHARED))
                ret = do_cow_fault(vmf);
        else
                ret = do_shared_fault(vmf);

        /* preallocated pagetable is unused: free it */
        if (vmf->prealloc_pte) {
                pte_free(vm_mm, vmf->prealloc_pte);
                vmf->prealloc_pte = NULL;
        }
        return ret;
}

int numa_migrate_check(struct folio *folio, struct vm_fault *vmf,
                      unsigned long addr, int *flags,
                      bool writable, int *last_cpupid)
{
        struct vm_area_struct *vma = vmf->vma;

        /*
         * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
         * much anyway since they can be in shared cache state. This misses
         * the case where a mapping is writable but the process never writes
         * to it but pte_write gets cleared during protection updates and
         * pte_dirty has unpredictable behaviour between PTE scan updates,
         * background writeback, dirty balancing and application behaviour.
         */
        if (!writable)
                *flags |= TNF_NO_GROUP;

        /*
         * Flag if the folio is shared between multiple address spaces. This
         * is later used when determining whether to group tasks together
         */
        if (folio_likely_mapped_shared(folio) && (vma->vm_flags & VM_SHARED))
                *flags |= TNF_SHARED;
        /*
         * For memory tiering mode, cpupid of slow memory page is used
         * to record page access time.  So use default value.
         */
        if (folio_use_access_time(folio))
                *last_cpupid = (-1 & LAST_CPUPID_MASK);
        else
                *last_cpupid = folio_last_cpupid(folio);

        /* Record the current PID acceesing VMA */
        vma_set_access_pid_bit(vma);

        count_vm_numa_event(NUMA_HINT_FAULTS);
#ifdef CONFIG_NUMA_BALANCING
        count_memcg_folio_events(folio, NUMA_HINT_FAULTS, 1);
#endif
        if (folio_nid(folio) == numa_node_id()) {
                count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
                *flags |= TNF_FAULT_LOCAL;
        }

        return mpol_misplaced(folio, vmf, addr);
}

static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_struct *vma,
                                        unsigned long fault_addr, pte_t *fault_pte,
                                        bool writable)
{
        pte_t pte, old_pte;

        old_pte = ptep_modify_prot_start(vma, fault_addr, fault_pte);
        pte = pte_modify(old_pte, vma->vm_page_prot);
        pte = pte_mkyoung(pte);
        if (writable)
                pte = pte_mkwrite(pte, vma);
        ptep_modify_prot_commit(vma, fault_addr, fault_pte, old_pte, pte);
        update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1);
}

static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma,
                                       struct folio *folio, pte_t fault_pte,
                                       bool ignore_writable, bool pte_write_upgrade)
{
        int nr = pte_pfn(fault_pte) - folio_pfn(folio);
        unsigned long start, end, addr = vmf->address;
        unsigned long addr_start = addr - (nr << PAGE_SHIFT);
        unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE);
        pte_t *start_ptep;

        /* Stay within the VMA and within the page table. */
        start = max3(addr_start, pt_start, vma->vm_start);
        end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE,
                   vma->vm_end);
        start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT);

        /* Restore all PTEs' mapping of the large folio */
        for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) {
                pte_t ptent = ptep_get(start_ptep);
                bool writable = false;

                if (!pte_present(ptent) || !pte_protnone(ptent))
                        continue;

                if (pfn_folio(pte_pfn(ptent)) != folio)
                        continue;

                if (!ignore_writable) {
                        ptent = pte_modify(ptent, vma->vm_page_prot);
                        writable = pte_write(ptent);
                        if (!writable && pte_write_upgrade &&
                            can_change_pte_writable(vma, addr, ptent))
                                writable = true;
                }

                numa_rebuild_single_mapping(vmf, vma, addr, start_ptep, writable);
        }
}

static vm_fault_t do_numa_page(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio = NULL;
        int nid = NUMA_NO_NODE;
        bool writable = false, ignore_writable = false;
        bool pte_write_upgrade = vma_wants_manual_pte_write_upgrade(vma);
        int last_cpupid;
        int target_nid;
        pte_t pte, old_pte;
        int flags = 0, nr_pages;

        /*
         * The pte cannot be used safely until we verify, while holding the page
         * table lock, that its contents have not changed during fault handling.
         */
        spin_lock(vmf->ptl);
        /* Read the live PTE from the page tables: */
        old_pte = ptep_get(vmf->pte);

        if (unlikely(!pte_same(old_pte, vmf->orig_pte))) {
                pte_unmap_unlock(vmf->pte, vmf->ptl);
                return 0;
        }

        pte = pte_modify(old_pte, vma->vm_page_prot);

        /*
         * Detect now whether the PTE could be writable; this information
         * is only valid while holding the PT lock.
         */
        writable = pte_write(pte);
        if (!writable && pte_write_upgrade &&
            can_change_pte_writable(vma, vmf->address, pte))
                writable = true;

        folio = vm_normal_folio(vma, vmf->address, pte);
        if (!folio || folio_is_zone_device(folio))
                goto out_map;

        nid = folio_nid(folio);
        nr_pages = folio_nr_pages(folio);

        target_nid = numa_migrate_check(folio, vmf, vmf->address, &flags,
                                        writable, &last_cpupid);
        if (target_nid == NUMA_NO_NODE)
                goto out_map;
        if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) {
                flags |= TNF_MIGRATE_FAIL;
                goto out_map;
        }
        /* The folio is isolated and isolation code holds a folio reference. */
        pte_unmap_unlock(vmf->pte, vmf->ptl);
        writable = false;
        ignore_writable = true;

        /* Migrate to the requested node */
        if (!migrate_misplaced_folio(folio, vma, target_nid)) {
                nid = target_nid;
                flags |= TNF_MIGRATED;
                task_numa_fault(last_cpupid, nid, nr_pages, flags);
                return 0;
        }

        flags |= TNF_MIGRATE_FAIL;
        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                       vmf->address, &vmf->ptl);
        if (unlikely(!vmf->pte))
                return 0;
        if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
                pte_unmap_unlock(vmf->pte, vmf->ptl);
                return 0;
        }
out_map:
        /*
         * Make it present again, depending on how arch implements
         * non-accessible ptes, some can allow access by kernel mode.
         */
        if (folio && folio_test_large(folio))
                numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable,
                                           pte_write_upgrade);
        else
                numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte,
                                            writable);
        pte_unmap_unlock(vmf->pte, vmf->ptl);

        if (nid != NUMA_NO_NODE)
                task_numa_fault(last_cpupid, nid, nr_pages, flags);
        return 0;
}

static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        if (vma_is_anonymous(vma))
                return do_huge_pmd_anonymous_page(vmf);
        if (vma->vm_ops->huge_fault)
                return vma->vm_ops->huge_fault(vmf, PMD_ORDER);
        return VM_FAULT_FALLBACK;
}

/* `inline' is required to avoid gcc 4.1.2 build error */
static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
        vm_fault_t ret;

        if (vma_is_anonymous(vma)) {
                if (likely(!unshare) &&
                    userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
                        if (userfaultfd_wp_async(vmf->vma))
                                goto split;
                        return handle_userfault(vmf, VM_UFFD_WP);
                }
                return do_huge_pmd_wp_page(vmf);
        }

        if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
                if (vma->vm_ops->huge_fault) {
                        ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
                        if (!(ret & VM_FAULT_FALLBACK))
                                return ret;
                }
        }

split:
        /* COW or write-notify handled on pte level: split pmd. */
        __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);

        return VM_FAULT_FALLBACK;
}

static vm_fault_t create_huge_pud(struct vm_fault *vmf)
{
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&                        \
        defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
        struct vm_area_struct *vma = vmf->vma;
        /* No support for anonymous transparent PUD pages yet */
        if (vma_is_anonymous(vma))
                return VM_FAULT_FALLBACK;
        if (vma->vm_ops->huge_fault)
                return vma->vm_ops->huge_fault(vmf, PUD_ORDER);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
        return VM_FAULT_FALLBACK;
}

static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
{
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&                        \
        defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
        struct vm_area_struct *vma = vmf->vma;
        vm_fault_t ret;

        /* No support for anonymous transparent PUD pages yet */
        if (vma_is_anonymous(vma))
                goto split;
        if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
                if (vma->vm_ops->huge_fault) {
                        ret = vma->vm_ops->huge_fault(vmf, PUD_ORDER);
                        if (!(ret & VM_FAULT_FALLBACK))
                                return ret;
                }
        }
split:
        /* COW or write-notify not handled on PUD level: split pud.*/
        __split_huge_pud(vma, vmf->pud, vmf->address);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
        return VM_FAULT_FALLBACK;
}

/*
 * These routines also need to handle stuff like marking pages dirty
 * and/or accessed for architectures that don't do it in hardware (most
 * RISC architectures).  The early dirtying is also good on the i386.
 *
 * There is also a hook called "update_mmu_cache()" that architectures
 * with external mmu caches can use to update those (ie the Sparc or
 * PowerPC hashed page tables that act as extended TLBs).
 *
 * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
 * concurrent faults).
 *
 * The mmap_lock may have been released depending on flags and our return value.
 * See filemap_fault() and __folio_lock_or_retry().
 */
static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
{
        pte_t entry;

        if (unlikely(pmd_none(*vmf->pmd))) {
                /*
                 * Leave __pte_alloc() until later: because vm_ops->fault may
                 * want to allocate huge page, and if we expose page table
                 * for an instant, it will be difficult to retract from
                 * concurrent faults and from rmap lookups.
                 */
                vmf->pte = NULL;
                vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID;
        } else {
                pmd_t dummy_pmdval;

                /*
                 * A regular pmd is established and it can't morph into a huge
                 * pmd by anon khugepaged, since that takes mmap_lock in write
                 * mode; but shmem or file collapse to THP could still morph
                 * it into a huge pmd: just retry later if so.
                 *
                 * Use the maywrite version to indicate that vmf->pte may be
                 * modified, but since we will use pte_same() to detect the
                 * change of the !pte_none() entry, there is no need to recheck
                 * the pmdval. Here we chooes to pass a dummy variable instead
                 * of NULL, which helps new user think about why this place is
                 * special.
                 */
                vmf->pte = pte_offset_map_rw_nolock(vmf->vma->vm_mm, vmf->pmd,
                                                    vmf->address, &dummy_pmdval,
                                                    &vmf->ptl);
                if (unlikely(!vmf->pte))
                        return 0;
                vmf->orig_pte = ptep_get_lockless(vmf->pte);
                vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID;

                if (pte_none(vmf->orig_pte)) {
                        pte_unmap(vmf->pte);
                        vmf->pte = NULL;
                }
        }

        if (!vmf->pte)
                return do_pte_missing(vmf);

        if (!pte_present(vmf->orig_pte))
                return do_swap_page(vmf);

        if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
                return do_numa_page(vmf);

        spin_lock(vmf->ptl);
        entry = vmf->orig_pte;
        if (unlikely(!pte_same(ptep_get(vmf->pte), entry))) {
                update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
                goto unlock;
        }
        if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
                if (!pte_write(entry))
                        return do_wp_page(vmf);
                else if (likely(vmf->flags & FAULT_FLAG_WRITE))
                        entry = pte_mkdirty(entry);
        }
        entry = pte_mkyoung(entry);
        if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
                                vmf->flags & FAULT_FLAG_WRITE)) {
                update_mmu_cache_range(vmf, vmf->vma, vmf->address,
                                vmf->pte, 1);
        } else {
                /* Skip spurious TLB flush for retried page fault */
                if (vmf->flags & FAULT_FLAG_TRIED)
                        goto unlock;
                /*
                 * This is needed only for protection faults but the arch code
                 * is not yet telling us if this is a protection fault or not.
                 * This still avoids useless tlb flushes for .text page faults
                 * with threads.
                 */
                if (vmf->flags & FAULT_FLAG_WRITE)
                        flush_tlb_fix_spurious_fault(vmf->vma, vmf->address,
                                                     vmf->pte);
        }
unlock:
        pte_unmap_unlock(vmf->pte, vmf->ptl);
        return 0;
}

/*
 * On entry, we hold either the VMA lock or the mmap_lock
 * (FAULT_FLAG_VMA_LOCK tells you which).  If VM_FAULT_RETRY is set in
 * the result, the mmap_lock is not held on exit.  See filemap_fault()
 * and __folio_lock_or_retry().
 */
static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
                unsigned long address, unsigned int flags)
{
        struct vm_fault vmf = {
                .vma = vma,
                .address = address & PAGE_MASK,
                .real_address = address,
                .flags = flags,
                .pgoff = linear_page_index(vma, address),
                .gfp_mask = __get_fault_gfp_mask(vma),
        };
        struct mm_struct *mm = vma->vm_mm;
        unsigned long vm_flags = vma->vm_flags;
        pgd_t *pgd;
        p4d_t *p4d;
        vm_fault_t ret;

        pgd = pgd_offset(mm, address);
        p4d = p4d_alloc(mm, pgd, address);
        if (!p4d)
                return VM_FAULT_OOM;

        vmf.pud = pud_alloc(mm, p4d, address);
        if (!vmf.pud)
                return VM_FAULT_OOM;
retry_pud:
        if (pud_none(*vmf.pud) &&
            thp_vma_allowable_order(vma, vm_flags,
                                TVA_IN_PF | TVA_ENFORCE_SYSFS, PUD_ORDER)) {
                ret = create_huge_pud(&vmf);
                if (!(ret & VM_FAULT_FALLBACK))
                        return ret;
        } else {
                pud_t orig_pud = *vmf.pud;

                barrier();
                if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {

                        /*
                         * TODO once we support anonymous PUDs: NUMA case and
                         * FAULT_FLAG_UNSHARE handling.
                         */
                        if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) {
                                ret = wp_huge_pud(&vmf, orig_pud);
                                if (!(ret & VM_FAULT_FALLBACK))
                                        return ret;
                        } else {
                                huge_pud_set_accessed(&vmf, orig_pud);
                                return 0;
                        }
                }
        }

        vmf.pmd = pmd_alloc(mm, vmf.pud, address);
        if (!vmf.pmd)
                return VM_FAULT_OOM;

        /* Huge pud page fault raced with pmd_alloc? */
        if (pud_trans_unstable(vmf.pud))
                goto retry_pud;

        if (pmd_none(*vmf.pmd) &&
            thp_vma_allowable_order(vma, vm_flags,
                                TVA_IN_PF | TVA_ENFORCE_SYSFS, PMD_ORDER)) {
                ret = create_huge_pmd(&vmf);
                if (!(ret & VM_FAULT_FALLBACK))
                        return ret;
        } else {
                vmf.orig_pmd = pmdp_get_lockless(vmf.pmd);

                if (unlikely(is_swap_pmd(vmf.orig_pmd))) {
                        VM_BUG_ON(thp_migration_supported() &&
                                          !is_pmd_migration_entry(vmf.orig_pmd));
                        if (is_pmd_migration_entry(vmf.orig_pmd))
                                pmd_migration_entry_wait(mm, vmf.pmd);
                        return 0;
                }
                if (pmd_trans_huge(vmf.orig_pmd) || pmd_devmap(vmf.orig_pmd)) {
                        if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
                                return do_huge_pmd_numa_page(&vmf);

                        if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
                            !pmd_write(vmf.orig_pmd)) {
                                ret = wp_huge_pmd(&vmf);
                                if (!(ret & VM_FAULT_FALLBACK))
                                        return ret;
                        } else {
                                huge_pmd_set_accessed(&vmf);
                                return 0;
                        }
                }
        }

        return handle_pte_fault(&vmf);
}

/**
 * mm_account_fault - Do page fault accounting
 * @mm: mm from which memcg should be extracted. It can be NULL.
 * @regs: the pt_regs struct pointer.  When set to NULL, will skip accounting
 *        of perf event counters, but we'll still do the per-task accounting to
 *        the task who triggered this page fault.
 * @address: the faulted address.
 * @flags: the fault flags.
 * @ret: the fault retcode.
 *
 * This will take care of most of the page fault accounting.  Meanwhile, it
 * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
 * updates.  However, note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
 * still be in per-arch page fault handlers at the entry of page fault.
 */
static inline void mm_account_fault(struct mm_struct *mm, struct pt_regs *regs,
                                    unsigned long address, unsigned int flags,
                                    vm_fault_t ret)
{
        bool major;

        /* Incomplete faults will be accounted upon completion. */
        if (ret & VM_FAULT_RETRY)
                return;

        /*
         * To preserve the behavior of older kernels, PGFAULT counters record
         * both successful and failed faults, as opposed to perf counters,
         * which ignore failed cases.
         */
        count_vm_event(PGFAULT);
        count_memcg_event_mm(mm, PGFAULT);

        /*
         * Do not account for unsuccessful faults (e.g. when the address wasn't
         * valid).  That includes arch_vma_access_permitted() failing before
         * reaching here. So this is not a "this many hardware page faults"
         * counter.  We should use the hw profiling for that.
         */
        if (ret & VM_FAULT_ERROR)
                return;

        /*
         * We define the fault as a major fault when the final successful fault
         * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
         * handle it immediately previously).
         */
        major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);

        if (major)
                current->maj_flt++;
        else
                current->min_flt++;

        /*
         * If the fault is done for GUP, regs will be NULL.  We only do the
         * accounting for the per thread fault counters who triggered the
         * fault, and we skip the perf event updates.
         */
        if (!regs)
                return;

        if (major)
                perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
        else
                perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
}

#ifdef CONFIG_LRU_GEN
static void lru_gen_enter_fault(struct vm_area_struct *vma)
{
        /* the LRU algorithm only applies to accesses with recency */
        current->in_lru_fault = vma_has_recency(vma);
}

static void lru_gen_exit_fault(void)
{
        current->in_lru_fault = false;
}
#else
static void lru_gen_enter_fault(struct vm_area_struct *vma)
{
}

static void lru_gen_exit_fault(void)
{
}
#endif /* CONFIG_LRU_GEN */

static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
                                       unsigned int *flags)
{
        if (unlikely(*flags & FAULT_FLAG_UNSHARE)) {
                if (WARN_ON_ONCE(*flags & FAULT_FLAG_WRITE))
                        return VM_FAULT_SIGSEGV;
                /*
                 * FAULT_FLAG_UNSHARE only applies to COW mappings. Let's
                 * just treat it like an ordinary read-fault otherwise.
                 */
                if (!is_cow_mapping(vma->vm_flags))
                        *flags &= ~FAULT_FLAG_UNSHARE;
        } else if (*flags & FAULT_FLAG_WRITE) {
                /* Write faults on read-only mappings are impossible ... */
                if (WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE)))
                        return VM_FAULT_SIGSEGV;
                /* ... and FOLL_FORCE only applies to COW mappings. */
                if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) &&
                                 !is_cow_mapping(vma->vm_flags)))
                        return VM_FAULT_SIGSEGV;
        }
#ifdef CONFIG_PER_VMA_LOCK
        /*
         * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of
         * the assumption that lock is dropped on VM_FAULT_RETRY.
         */
        if (WARN_ON_ONCE((*flags &
                        (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) ==
                        (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)))
                return VM_FAULT_SIGSEGV;
#endif

        return 0;
}

/*
 * By the time we get here, we already hold the mm semaphore
 *
 * The mmap_lock may have been released depending on flags and our
 * return value.  See filemap_fault() and __folio_lock_or_retry().
 */
vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
                           unsigned int flags, struct pt_regs *regs)
{
        /* If the fault handler drops the mmap_lock, vma may be freed */
        struct mm_struct *mm = vma->vm_mm;
        vm_fault_t ret;
        bool is_droppable;

        __set_current_state(TASK_RUNNING);

        ret = sanitize_fault_flags(vma, &flags);
        if (ret)
                goto out;

        if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
                                            flags & FAULT_FLAG_INSTRUCTION,
                                            flags & FAULT_FLAG_REMOTE)) {
                ret = VM_FAULT_SIGSEGV;
                goto out;
        }

        is_droppable = !!(vma->vm_flags & VM_DROPPABLE);

        /*
         * Enable the memcg OOM handling for faults triggered in user
         * space.  Kernel faults are handled more gracefully.
         */
        if (flags & FAULT_FLAG_USER)
                mem_cgroup_enter_user_fault();

        lru_gen_enter_fault(vma);

        if (unlikely(is_vm_hugetlb_page(vma)))
                ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
        else
                ret = __handle_mm_fault(vma, address, flags);

        /*
         * Warning: It is no longer safe to dereference vma-> after this point,
         * because mmap_lock might have been dropped by __handle_mm_fault(), so
         * vma might be destroyed from underneath us.
         */

        lru_gen_exit_fault();

        /* If the mapping is droppable, then errors due to OOM aren't fatal. */
        if (is_droppable)
                ret &= ~VM_FAULT_OOM;

        if (flags & FAULT_FLAG_USER) {
                mem_cgroup_exit_user_fault();
                /*
                 * The task may have entered a memcg OOM situation but
                 * if the allocation error was handled gracefully (no
                 * VM_FAULT_OOM), there is no need to kill anything.
                 * Just clean up the OOM state peacefully.
                 */
                if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
                        mem_cgroup_oom_synchronize(false);
        }
out:
        mm_account_fault(mm, regs, address, flags, ret);

        return ret;
}
EXPORT_SYMBOL_GPL(handle_mm_fault);

#ifdef CONFIG_LOCK_MM_AND_FIND_VMA
#include <linux/extable.h>

static inline bool get_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
{
        if (likely(mmap_read_trylock(mm)))
                return true;

        if (regs && !user_mode(regs)) {
                unsigned long ip = exception_ip(regs);
                if (!search_exception_tables(ip))
                        return false;
        }

        return !mmap_read_lock_killable(mm);
}

static inline bool mmap_upgrade_trylock(struct mm_struct *mm)
{
        /*
         * We don't have this operation yet.
         *
         * It should be easy enough to do: it's basically a
         *    atomic_long_try_cmpxchg_acquire()
         * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but
         * it also needs the proper lockdep magic etc.
         */
        return false;
}

static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
{
        mmap_read_unlock(mm);
        if (regs && !user_mode(regs)) {
                unsigned long ip = exception_ip(regs);
                if (!search_exception_tables(ip))
                        return false;
        }
        return !mmap_write_lock_killable(mm);
}

/*
 * Helper for page fault handling.
 *
 * This is kind of equivalend to "mmap_read_lock()" followed
 * by "find_extend_vma()", except it's a lot more careful about
 * the locking (and will drop the lock on failure).
 *
 * For example, if we have a kernel bug that causes a page
 * fault, we don't want to just use mmap_read_lock() to get
 * the mm lock, because that would deadlock if the bug were
 * to happen while we're holding the mm lock for writing.
 *
 * So this checks the exception tables on kernel faults in
 * order to only do this all for instructions that are actually
 * expected to fault.
 *
 * We can also actually take the mm lock for writing if we
 * need to extend the vma, which helps the VM layer a lot.
 */
struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
                        unsigned long addr, struct pt_regs *regs)
{
        struct vm_area_struct *vma;

        if (!get_mmap_lock_carefully(mm, regs))
                return NULL;

        vma = find_vma(mm, addr);
        if (likely(vma && (vma->vm_start <= addr)))
                return vma;

        /*
         * Well, dang. We might still be successful, but only
         * if we can extend a vma to do so.
         */
        if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) {
                mmap_read_unlock(mm);
                return NULL;
        }

        /*
         * We can try to upgrade the mmap lock atomically,
         * in which case we can continue to use the vma
         * we already looked up.
         *
         * Otherwise we'll have to drop the mmap lock and
         * re-take it, and also look up the vma again,
         * re-checking it.
         */
        if (!mmap_upgrade_trylock(mm)) {
                if (!upgrade_mmap_lock_carefully(mm, regs))
                        return NULL;

                vma = find_vma(mm, addr);
                if (!vma)
                        goto fail;
                if (vma->vm_start <= addr)
                        goto success;
                if (!(vma->vm_flags & VM_GROWSDOWN))
                        goto fail;
        }

        if (expand_stack_locked(vma, addr))
                goto fail;

success:
        mmap_write_downgrade(mm);
        return vma;

fail:
        mmap_write_unlock(mm);
        return NULL;
}
#endif

#ifdef CONFIG_PER_VMA_LOCK
/*
 * Lookup and lock a VMA under RCU protection. Returned VMA is guaranteed to be
 * stable and not isolated. If the VMA is not found or is being modified the
 * function returns NULL.
 */
struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
                                          unsigned long address)
{
        MA_STATE(mas, &mm->mm_mt, address, address);
        struct vm_area_struct *vma;

        rcu_read_lock();
retry:
        vma = mas_walk(&mas);
        if (!vma)
                goto inval;

        if (!vma_start_read(vma))
                goto inval;

        /* Check if the VMA got isolated after we found it */
        if (vma->detached) {
                vma_end_read(vma);
                count_vm_vma_lock_event(VMA_LOCK_MISS);
                /* The area was replaced with another one */
                goto retry;
        }
        /*
         * At this point, we have a stable reference to a VMA: The VMA is
         * locked and we know it hasn't already been isolated.
         * From here on, we can access the VMA without worrying about which
         * fields are accessible for RCU readers.
         */

        /* Check since vm_start/vm_end might change before we lock the VMA */
        if (unlikely(address < vma->vm_start || address >= vma->vm_end))
                goto inval_end_read;

        rcu_read_unlock();
        return vma;

inval_end_read:
        vma_end_read(vma);
inval:
        rcu_read_unlock();
        count_vm_vma_lock_event(VMA_LOCK_ABORT);
        return NULL;
}
#endif /* CONFIG_PER_VMA_LOCK */

#ifndef __PAGETABLE_P4D_FOLDED
/*
 * Allocate p4d page table.
 * We've already handled the fast-path in-line.
 */
int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
{
        p4d_t *new = p4d_alloc_one(mm, address);
        if (!new)
                return -ENOMEM;

        spin_lock(&mm->page_table_lock);
        if (pgd_present(*pgd)) {        /* Another has populated it */
                p4d_free(mm, new);
        } else {
                smp_wmb(); /* See comment in pmd_install() */
                pgd_populate(mm, pgd, new);
        }
        spin_unlock(&mm->page_table_lock);
        return 0;
}
#endif /* __PAGETABLE_P4D_FOLDED */

#ifndef __PAGETABLE_PUD_FOLDED
/*
 * Allocate page upper directory.
 * We've already handled the fast-path in-line.
 */
int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
{
        pud_t *new = pud_alloc_one(mm, address);
        if (!new)
                return -ENOMEM;

        spin_lock(&mm->page_table_lock);
        if (!p4d_present(*p4d)) {
                mm_inc_nr_puds(mm);
                smp_wmb(); /* See comment in pmd_install() */
                p4d_populate(mm, p4d, new);
        } else        /* Another has populated it */
                pud_free(mm, new);
        spin_unlock(&mm->page_table_lock);
        return 0;
}
#endif /* __PAGETABLE_PUD_FOLDED */

#ifndef __PAGETABLE_PMD_FOLDED
/*
 * Allocate page middle directory.
 * We've already handled the fast-path in-line.
 */
int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
{
        spinlock_t *ptl;
        pmd_t *new = pmd_alloc_one(mm, address);
        if (!new)
                return -ENOMEM;

        ptl = pud_lock(mm, pud);
        if (!pud_present(*pud)) {
                mm_inc_nr_pmds(mm);
                smp_wmb(); /* See comment in pmd_install() */
                pud_populate(mm, pud, new);
        } else {        /* Another has populated it */
                pmd_free(mm, new);
        }
        spin_unlock(ptl);
        return 0;
}
#endif /* __PAGETABLE_PMD_FOLDED */

static inline void pfnmap_args_setup(struct follow_pfnmap_args *args,
                                     spinlock_t *lock, pte_t *ptep,
                                     pgprot_t pgprot, unsigned long pfn_base,
                                     unsigned long addr_mask, bool writable,
                                     bool special)
{
        args->lock = lock;
        args->ptep = ptep;
        args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT);
        args->pgprot = pgprot;
        args->writable = writable;
        args->special = special;
}

static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma)
{
#ifdef CONFIG_LOCKDEP
        struct file *file = vma->vm_file;
        struct address_space *mapping = file ? file->f_mapping : NULL;

        if (mapping)
                lockdep_assert(lockdep_is_held(&mapping->i_mmap_rwsem) ||
                               lockdep_is_held(&vma->vm_mm->mmap_lock));
        else
                lockdep_assert(lockdep_is_held(&vma->vm_mm->mmap_lock));
#endif
}

/**
 * follow_pfnmap_start() - Look up a pfn mapping at a user virtual address
 * @args: Pointer to struct @follow_pfnmap_args
 *
 * The caller needs to setup args->vma and args->address to point to the
 * virtual address as the target of such lookup.  On a successful return,
 * the results will be put into other output fields.
 *
 * After the caller finished using the fields, the caller must invoke
 * another follow_pfnmap_end() to proper releases the locks and resources
 * of such look up request.
 *
 * During the start() and end() calls, the results in @args will be valid
 * as proper locks will be held.  After the end() is called, all the fields
 * in @follow_pfnmap_args will be invalid to be further accessed.  Further
 * use of such information after end() may require proper synchronizations
 * by the caller with page table updates, otherwise it can create a
 * security bug.
 *
 * If the PTE maps a refcounted page, callers are responsible to protect
 * against invalidation with MMU notifiers; otherwise access to the PFN at
 * a later point in time can trigger use-after-free.
 *
 * Only IO mappings and raw PFN mappings are allowed.  The mmap semaphore
 * should be taken for read, and the mmap semaphore cannot be released
 * before the end() is invoked.
 *
 * This function must not be used to modify PTE content.
 *
 * Return: zero on success, negative otherwise.
 */
int follow_pfnmap_start(struct follow_pfnmap_args *args)
{
        struct vm_area_struct *vma = args->vma;
        unsigned long address = args->address;
        struct mm_struct *mm = vma->vm_mm;
        spinlock_t *lock;
        pgd_t *pgdp;
        p4d_t *p4dp, p4d;
        pud_t *pudp, pud;
        pmd_t *pmdp, pmd;
        pte_t *ptep, pte;

        pfnmap_lockdep_assert(vma);

        if (unlikely(address < vma->vm_start || address >= vma->vm_end))
                goto out;

        if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
                goto out;
retry:
        pgdp = pgd_offset(mm, address);
        if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
                goto out;

        p4dp = p4d_offset(pgdp, address);
        p4d = READ_ONCE(*p4dp);
        if (p4d_none(p4d) || unlikely(p4d_bad(p4d)))
                goto out;

        pudp = pud_offset(p4dp, address);
        pud = READ_ONCE(*pudp);
        if (pud_none(pud))
                goto out;
        if (pud_leaf(pud)) {
                lock = pud_lock(mm, pudp);
                if (!unlikely(pud_leaf(pud))) {
                        spin_unlock(lock);
                        goto retry;
                }
                pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud),
                                  pud_pfn(pud), PUD_MASK, pud_write(pud),
                                  pud_special(pud));
                return 0;
        }

        pmdp = pmd_offset(pudp, address);
        pmd = pmdp_get_lockless(pmdp);
        if (pmd_leaf(pmd)) {
                lock = pmd_lock(mm, pmdp);
                if (!unlikely(pmd_leaf(pmd))) {
                        spin_unlock(lock);
                        goto retry;
                }
                pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd),
                                  pmd_pfn(pmd), PMD_MASK, pmd_write(pmd),
                                  pmd_special(pmd));
                return 0;
        }

        ptep = pte_offset_map_lock(mm, pmdp, address, &lock);
        if (!ptep)
                goto out;
        pte = ptep_get(ptep);
        if (!pte_present(pte))
                goto unlock;
        pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte),
                          pte_pfn(pte), PAGE_MASK, pte_write(pte),
                          pte_special(pte));
        return 0;
unlock:
        pte_unmap_unlock(ptep, lock);
out:
        return -EINVAL;
}
EXPORT_SYMBOL_GPL(follow_pfnmap_start);

/**
 * follow_pfnmap_end(): End a follow_pfnmap_start() process
 * @args: Pointer to struct @follow_pfnmap_args
 *
 * Must be used in pair of follow_pfnmap_start().  See the start() function
 * above for more information.
 */
void follow_pfnmap_end(struct follow_pfnmap_args *args)
{
        if (args->lock)
                spin_unlock(args->lock);
        if (args->ptep)
                pte_unmap(args->ptep);
}
EXPORT_SYMBOL_GPL(follow_pfnmap_end);

#ifdef CONFIG_HAVE_IOREMAP_PROT
/**
 * generic_access_phys - generic implementation for iomem mmap access
 * @vma: the vma to access
 * @addr: userspace address, not relative offset within @vma
 * @buf: buffer to read/write
 * @len: length of transfer
 * @write: set to FOLL_WRITE when writing, otherwise reading
 *
 * This is a generic implementation for &vm_operations_struct.access for an
 * iomem mapping. This callback is used by access_process_vm() when the @vma is
 * not page based.
 */
int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
                        void *buf, int len, int write)
{
        resource_size_t phys_addr;
        unsigned long prot = 0;
        void __iomem *maddr;
        int offset = offset_in_page(addr);
        int ret = -EINVAL;
        bool writable;
        struct follow_pfnmap_args args = { .vma = vma, .address = addr };

retry:
        if (follow_pfnmap_start(&args))
                return -EINVAL;
        prot = pgprot_val(args.pgprot);
        phys_addr = (resource_size_t)args.pfn << PAGE_SHIFT;
        writable = args.writable;
        follow_pfnmap_end(&args);

        if ((write & FOLL_WRITE) && !writable)
                return -EINVAL;

        maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
        if (!maddr)
                return -ENOMEM;

        if (follow_pfnmap_start(&args))
                goto out_unmap;

        if ((prot != pgprot_val(args.pgprot)) ||
            (phys_addr != (args.pfn << PAGE_SHIFT)) ||
            (writable != args.writable)) {
                follow_pfnmap_end(&args);
                iounmap(maddr);
                goto retry;
        }

        if (write)
                memcpy_toio(maddr + offset, buf, len);
        else
                memcpy_fromio(buf, maddr + offset, len);
        ret = len;
        follow_pfnmap_end(&args);
out_unmap:
        iounmap(maddr);

        return ret;
}
EXPORT_SYMBOL_GPL(generic_access_phys);
#endif

/*
 * Access another process' address space as given in mm.
 */
static int __access_remote_vm(struct mm_struct *mm, unsigned long addr,
                              void *buf, int len, unsigned int gup_flags)
{
        void *old_buf = buf;
        int write = gup_flags & FOLL_WRITE;

        if (mmap_read_lock_killable(mm))
                return 0;

        /* Untag the address before looking up the VMA */
        addr = untagged_addr_remote(mm, addr);

        /* Avoid triggering the temporary warning in __get_user_pages */
        if (!vma_lookup(mm, addr) && !expand_stack(mm, addr))
                return 0;

        /* ignore errors, just check how much was successfully transferred */
        while (len) {
                int bytes, offset;
                void *maddr;
                struct vm_area_struct *vma = NULL;
                struct page *page = get_user_page_vma_remote(mm, addr,
                                                             gup_flags, &vma);

                if (IS_ERR(page)) {
                        /* We might need to expand the stack to access it */
                        vma = vma_lookup(mm, addr);
                        if (!vma) {
                                vma = expand_stack(mm, addr);

                                /* mmap_lock was dropped on failure */
                                if (!vma)
                                        return buf - old_buf;

                                /* Try again if stack expansion worked */
                                continue;
                        }

                        /*
                         * Check if this is a VM_IO | VM_PFNMAP VMA, which
                         * we can access using slightly different code.
                         */
                        bytes = 0;
#ifdef CONFIG_HAVE_IOREMAP_PROT
                        if (vma->vm_ops && vma->vm_ops->access)
                                bytes = vma->vm_ops->access(vma, addr, buf,
                                                            len, write);
#endif
                        if (bytes <= 0)
                                break;
                } else {
                        bytes = len;
                        offset = addr & (PAGE_SIZE-1);
                        if (bytes > PAGE_SIZE-offset)
                                bytes = PAGE_SIZE-offset;

                        maddr = kmap_local_page(page);
                        if (write) {
                                copy_to_user_page(vma, page, addr,
                                                  maddr + offset, buf, bytes);
                                set_page_dirty_lock(page);
                        } else {
                                copy_from_user_page(vma, page, addr,
                                                    buf, maddr + offset, bytes);
                        }
                        unmap_and_put_page(page, maddr);
                }
                len -= bytes;
                buf += bytes;
                addr += bytes;
        }
        mmap_read_unlock(mm);

        return buf - old_buf;
}

/**
 * access_remote_vm - access another process' address space
 * @mm:                the mm_struct of the target address space
 * @addr:        start address to access
 * @buf:        source or destination buffer
 * @len:        number of bytes to transfer
 * @gup_flags:        flags modifying lookup behaviour
 *
 * The caller must hold a reference on @mm.
 *
 * Return: number of bytes copied from source to destination.
 */
int access_remote_vm(struct mm_struct *mm, unsigned long addr,
                void *buf, int len, unsigned int gup_flags)
{
        return __access_remote_vm(mm, addr, buf, len, gup_flags);
}

/*
 * Access another process' address space.
 * Source/target buffer must be kernel space,
 * Do not walk the page table directly, use get_user_pages
 */
int access_process_vm(struct task_struct *tsk, unsigned long addr,
                void *buf, int len, unsigned int gup_flags)
{
        struct mm_struct *mm;
        int ret;

        mm = get_task_mm(tsk);
        if (!mm)
                return 0;

        ret = __access_remote_vm(mm, addr, buf, len, gup_flags);

        mmput(mm);

        return ret;
}
EXPORT_SYMBOL_GPL(access_process_vm);

/*
 * Print the name of a VMA.
 */
void print_vma_addr(char *prefix, unsigned long ip)
{
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;

        /*
         * we might be running from an atomic context so we cannot sleep
         */
        if (!mmap_read_trylock(mm))
                return;

        vma = vma_lookup(mm, ip);
        if (vma && vma->vm_file) {
                struct file *f = vma->vm_file;
                ip -= vma->vm_start;
                ip += vma->vm_pgoff << PAGE_SHIFT;
                printk("%s%pD[%lx,%lx+%lx]", prefix, f, ip,
                                vma->vm_start,
                                vma->vm_end - vma->vm_start);
        }
        mmap_read_unlock(mm);
}

#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
void __might_fault(const char *file, int line)
{
        if (pagefault_disabled())
                return;
        __might_sleep(file, line);
#if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
        if (current->mm)
                might_lock_read(&current->mm->mmap_lock);
#endif
}
EXPORT_SYMBOL(__might_fault);
#endif

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
/*
 * Process all subpages of the specified huge page with the specified
 * operation.  The target subpage will be processed last to keep its
 * cache lines hot.
 */
static inline int process_huge_page(
        unsigned long addr_hint, unsigned int nr_pages,
        int (*process_subpage)(unsigned long addr, int idx, void *arg),
        void *arg)
{
        int i, n, base, l, ret;
        unsigned long addr = addr_hint &
                ~(((unsigned long)nr_pages << PAGE_SHIFT) - 1);

        /* Process target subpage last to keep its cache lines hot */
        might_sleep();
        n = (addr_hint - addr) / PAGE_SIZE;
        if (2 * n <= nr_pages) {
                /* If target subpage in first half of huge page */
                base = 0;
                l = n;
                /* Process subpages at the end of huge page */
                for (i = nr_pages - 1; i >= 2 * n; i--) {
                        cond_resched();
                        ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
                        if (ret)
                                return ret;
                }
        } else {
                /* If target subpage in second half of huge page */
                base = nr_pages - 2 * (nr_pages - n);
                l = nr_pages - n;
                /* Process subpages at the begin of huge page */
                for (i = 0; i < base; i++) {
                        cond_resched();
                        ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
                        if (ret)
                                return ret;
                }
        }
        /*
         * Process remaining subpages in left-right-left-right pattern
         * towards the target subpage
         */
        for (i = 0; i < l; i++) {
                int left_idx = base + i;
                int right_idx = base + 2 * l - 1 - i;

                cond_resched();
                ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
                if (ret)
                        return ret;
                cond_resched();
                ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
                if (ret)
                        return ret;
        }
        return 0;
}

static void clear_gigantic_page(struct folio *folio, unsigned long addr,
                                unsigned int nr_pages)
{
        int i;

        might_sleep();
        for (i = 0; i < nr_pages; i++) {
                cond_resched();
                clear_user_highpage(folio_page(folio, i), addr + i * PAGE_SIZE);
        }
}

static int clear_subpage(unsigned long addr, int idx, void *arg)
{
        struct folio *folio = arg;

        clear_user_highpage(folio_page(folio, idx), addr);
        return 0;
}

/**
 * folio_zero_user - Zero a folio which will be mapped to userspace.
 * @folio: The folio to zero.
 * @addr_hint: The address will be accessed or the base address if uncelar.
 */
void folio_zero_user(struct folio *folio, unsigned long addr_hint)
{
        unsigned int nr_pages = folio_nr_pages(folio);

        if (unlikely(nr_pages > MAX_ORDER_NR_PAGES))
                clear_gigantic_page(folio, addr_hint, nr_pages);
        else
                process_huge_page(addr_hint, nr_pages, clear_subpage, folio);
}

static int copy_user_gigantic_page(struct folio *dst, struct folio *src,
                                   unsigned long addr,
                                   struct vm_area_struct *vma,
                                   unsigned int nr_pages)
{
        int i;
        struct page *dst_page;
        struct page *src_page;

        for (i = 0; i < nr_pages; i++) {
                dst_page = folio_page(dst, i);
                src_page = folio_page(src, i);

                cond_resched();
                if (copy_mc_user_highpage(dst_page, src_page,
                                          addr + i*PAGE_SIZE, vma))
                        return -EHWPOISON;
        }
        return 0;
}

struct copy_subpage_arg {
        struct folio *dst;
        struct folio *src;
        struct vm_area_struct *vma;
};

static int copy_subpage(unsigned long addr, int idx, void *arg)
{
        struct copy_subpage_arg *copy_arg = arg;
        struct page *dst = folio_page(copy_arg->dst, idx);
        struct page *src = folio_page(copy_arg->src, idx);

        if (copy_mc_user_highpage(dst, src, addr, copy_arg->vma))
                return -EHWPOISON;
        return 0;
}

int copy_user_large_folio(struct folio *dst, struct folio *src,
                          unsigned long addr_hint, struct vm_area_struct *vma)
{
        unsigned int nr_pages = folio_nr_pages(dst);
        struct copy_subpage_arg arg = {
                .dst = dst,
                .src = src,
                .vma = vma,
        };

        if (unlikely(nr_pages > MAX_ORDER_NR_PAGES))
                return copy_user_gigantic_page(dst, src, addr_hint, vma, nr_pages);

        return process_huge_page(addr_hint, nr_pages, copy_subpage, &arg);
}

long copy_folio_from_user(struct folio *dst_folio,
                           const void __user *usr_src,
                           bool allow_pagefault)
{
        void *kaddr;
        unsigned long i, rc = 0;
        unsigned int nr_pages = folio_nr_pages(dst_folio);
        unsigned long ret_val = nr_pages * PAGE_SIZE;
        struct page *subpage;

        for (i = 0; i < nr_pages; i++) {
                subpage = folio_page(dst_folio, i);
                kaddr = kmap_local_page(subpage);
                if (!allow_pagefault)
                        pagefault_disable();
                rc = copy_from_user(kaddr, usr_src + i * PAGE_SIZE, PAGE_SIZE);
                if (!allow_pagefault)
                        pagefault_enable();
                kunmap_local(kaddr);

                ret_val -= (PAGE_SIZE - rc);
                if (rc)
                        break;

                flush_dcache_page(subpage);

                cond_resched();
        }
        return ret_val;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */

#if defined(CONFIG_SPLIT_PTE_PTLOCKS) && ALLOC_SPLIT_PTLOCKS

static struct kmem_cache *page_ptl_cachep;

void __init ptlock_cache_init(void)
{
        page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
                        SLAB_PANIC, NULL);
}

bool ptlock_alloc(struct ptdesc *ptdesc)
{
        spinlock_t *ptl;

        ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
        if (!ptl)
                return false;
        ptdesc->ptl = ptl;
        return true;
}

void ptlock_free(struct ptdesc *ptdesc)
{
        kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
}
#endif

void vma_pgtable_walk_begin(struct vm_area_struct *vma)
{
        if (is_vm_hugetlb_page(vma))
                hugetlb_vma_lock_read(vma);
}

void vma_pgtable_walk_end(struct vm_area_struct *vma)
{
        if (is_vm_hugetlb_page(vma))
                hugetlb_vma_unlock_read(vma);
}



































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* Definitions for key type implementations
 *
 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#ifndef _LINUX_KEY_TYPE_H
#define _LINUX_KEY_TYPE_H

#include <linux/key.h>
#include <linux/errno.h>

#ifdef CONFIG_KEYS

struct kernel_pkey_query;
struct kernel_pkey_params;

/*
 * Pre-parsed payload, used by key add, update and instantiate.
 *
 * This struct will be cleared and data and datalen will be set with the data
 * and length parameters from the caller and quotalen will be set from
 * def_datalen from the key type.  Then if the preparse() op is provided by the
 * key type, that will be called.  Then the struct will be passed to the
 * instantiate() or the update() op.
 *
 * If the preparse() op is given, the free_preparse() op will be called to
 * clear the contents.
 */
struct key_preparsed_payload {
        const char        *orig_description; /* Actual or proposed description (maybe NULL) */
        char                *description;        /* Proposed key description (or NULL) */
        union key_payload payload;        /* Proposed payload */
        const void        *data;                /* Raw data */
        size_t                datalen;        /* Raw datalen */
        size_t                quotalen;        /* Quota length for proposed payload */
        time64_t        expiry;                /* Expiry time of key */
} __randomize_layout;

typedef int (*request_key_actor_t)(struct key *auth_key, void *aux);

/*
 * Preparsed matching criterion.
 */
struct key_match_data {
        /* Comparison function, defaults to exact description match, but can be
         * overridden by type->match_preparse().  Should return true if a match
         * is found and false if not.
         */
        bool (*cmp)(const struct key *key,
                    const struct key_match_data *match_data);

        const void        *raw_data;        /* Raw match data */
        void                *preparsed;        /* For ->match_preparse() to stash stuff */
        unsigned        lookup_type;        /* Type of lookup for this search. */
#define KEYRING_SEARCH_LOOKUP_DIRECT        0x0000        /* Direct lookup by description. */
#define KEYRING_SEARCH_LOOKUP_ITERATE        0x0001        /* Iterative search. */
};

/*
 * kernel managed key type definition
 */
struct key_type {
        /* name of the type */
        const char *name;

        /* default payload length for quota precalculation (optional)
         * - this can be used instead of calling key_payload_reserve(), that
         *   function only needs to be called if the real datalen is different
         */
        size_t def_datalen;

        unsigned int flags;
#define KEY_TYPE_NET_DOMAIN        0x00000001 /* Keys of this type have a net namespace domain */
#define KEY_TYPE_INSTANT_REAP        0x00000002 /* Keys of this type don't have a delay after expiring */

        /* vet a description */
        int (*vet_description)(const char *description);

        /* Preparse the data blob from userspace that is to be the payload,
         * generating a proposed description and payload that will be handed to
         * the instantiate() and update() ops.
         */
        int (*preparse)(struct key_preparsed_payload *prep);

        /* Free a preparse data structure.
         */
        void (*free_preparse)(struct key_preparsed_payload *prep);

        /* instantiate a key of this type
         * - this method should call key_payload_reserve() to determine if the
         *   user's quota will hold the payload
         */
        int (*instantiate)(struct key *key, struct key_preparsed_payload *prep);

        /* update a key of this type (optional)
         * - this method should call key_payload_reserve() to recalculate the
         *   quota consumption
         * - the key must be locked against read when modifying
         */
        int (*update)(struct key *key, struct key_preparsed_payload *prep);

        /* Preparse the data supplied to ->match() (optional).  The
         * data to be preparsed can be found in match_data->raw_data.
         * The lookup type can also be set by this function.
         */
        int (*match_preparse)(struct key_match_data *match_data);

        /* Free preparsed match data (optional).  This should be supplied it
         * ->match_preparse() is supplied. */
        void (*match_free)(struct key_match_data *match_data);

        /* clear some of the data from a key on revokation (optional)
         * - the key's semaphore will be write-locked by the caller
         */
        void (*revoke)(struct key *key);

        /* clear the data from a key (optional) */
        void (*destroy)(struct key *key);

        /* describe a key */
        void (*describe)(const struct key *key, struct seq_file *p);

        /* read a key's data (optional)
         * - permission checks will be done by the caller
         * - the key's semaphore will be readlocked by the caller
         * - should return the amount of data that could be read, no matter how
         *   much is copied into the buffer
         * - shouldn't do the copy if the buffer is NULL
         */
        long (*read)(const struct key *key, char *buffer, size_t buflen);

        /* handle request_key() for this type instead of invoking
         * /sbin/request-key (optional)
         * - key is the key to instantiate
         * - authkey is the authority to assume when instantiating this key
         * - op is the operation to be done, usually "create"
         * - the call must not return until the instantiation process has run
         *   its course
         */
        request_key_actor_t request_key;

        /* Look up a keyring access restriction (optional)
         *
         * - NULL is a valid return value (meaning the requested restriction
         *   is known but will never block addition of a key)
         * - should return -EINVAL if the restriction is unknown
         */
        struct key_restriction *(*lookup_restriction)(const char *params);

        /* Asymmetric key accessor functions. */
        int (*asym_query)(const struct kernel_pkey_params *params,
                          struct kernel_pkey_query *info);
        int (*asym_eds_op)(struct kernel_pkey_params *params,
                           const void *in, void *out);
        int (*asym_verify_signature)(struct kernel_pkey_params *params,
                                     const void *in, const void *in2);

        /* internal fields */
        struct list_head        link;                /* link in types list */
        struct lock_class_key        lock_class;        /* key->sem lock class */
} __randomize_layout;

extern struct key_type key_type_keyring;

extern int register_key_type(struct key_type *ktype);
extern void unregister_key_type(struct key_type *ktype);

extern int key_payload_reserve(struct key *key, size_t datalen);
extern int key_instantiate_and_link(struct key *key,
                                    const void *data,
                                    size_t datalen,
                                    struct key *keyring,
                                    struct key *authkey);
extern int key_reject_and_link(struct key *key,
                               unsigned timeout,
                               unsigned error,
                               struct key *keyring,
                               struct key *authkey);
extern void complete_request_key(struct key *authkey, int error);

static inline int key_negate_and_link(struct key *key,
                                      unsigned timeout,
                                      struct key *keyring,
                                      struct key *authkey)
{
        return key_reject_and_link(key, timeout, ENOKEY, keyring, authkey);
}

extern int generic_key_instantiate(struct key *key, struct key_preparsed_payload *prep);

#endif /* CONFIG_KEYS */
#endif /* _LINUX_KEY_TYPE_H */




















































































































    1 

   38 




   60 




























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
 * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
 * Copyright (c) 2005 Intel Corporation.  All rights reserved.
 */

#ifndef IB_ADDR_H
#define IB_ADDR_H

#include <linux/ethtool.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/if_arp.h>
#include <linux/netdevice.h>
#include <linux/inetdevice.h>
#include <linux/socket.h>
#include <linux/if_vlan.h>
#include <net/ipv6.h>
#include <net/if_inet6.h>
#include <net/ip.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_pack.h>
#include <net/net_namespace.h>

/**
 * struct rdma_dev_addr - Contains resolved RDMA hardware addresses
 * @src_dev_addr:        Source MAC address.
 * @dst_dev_addr:        Destination MAC address.
 * @broadcast:                Broadcast address of the device.
 * @dev_type:                The interface hardware type of the device.
 * @bound_dev_if:        An optional device interface index.
 * @transport:                The transport type used.
 * @net:                Network namespace containing the bound_dev_if net_dev.
 * @sgid_attr:                GID attribute to use for identified SGID
 */
struct rdma_dev_addr {
        unsigned char src_dev_addr[MAX_ADDR_LEN];
        unsigned char dst_dev_addr[MAX_ADDR_LEN];
        unsigned char broadcast[MAX_ADDR_LEN];
        unsigned short dev_type;
        int bound_dev_if;
        enum rdma_transport_type transport;
        struct net *net;
        const struct ib_gid_attr *sgid_attr;
        enum rdma_network_type network;
        int hoplimit;
};

/**
 * rdma_translate_ip - Translate a local IP address to an RDMA hardware
 *   address.
 *
 * The dev_addr->net field must be initialized.
 */
int rdma_translate_ip(const struct sockaddr *addr,
                      struct rdma_dev_addr *dev_addr);

/**
 * rdma_resolve_ip - Resolve source and destination IP addresses to
 *   RDMA hardware addresses.
 * @src_addr: An optional source address to use in the resolution.  If a
 *   source address is not provided, a usable address will be returned via
 *   the callback.
 * @dst_addr: The destination address to resolve.
 * @addr: A reference to a data location that will receive the resolved
 *   addresses.  The data location must remain valid until the callback has
 *   been invoked. The net field of the addr struct must be valid.
 * @timeout_ms: Amount of time to wait for the address resolution to complete.
 * @callback: Call invoked once address resolution has completed, timed out,
 *   or been canceled.  A status of 0 indicates success.
 * @resolve_by_gid_attr:        Resolve the ip based on the GID attribute from
 *                                rdma_dev_addr.
 * @context: User-specified context associated with the call.
 */
int rdma_resolve_ip(struct sockaddr *src_addr, const struct sockaddr *dst_addr,
                    struct rdma_dev_addr *addr, unsigned long timeout_ms,
                    void (*callback)(int status, struct sockaddr *src_addr,
                                     struct rdma_dev_addr *addr, void *context),
                    bool resolve_by_gid_attr, void *context);

void rdma_addr_cancel(struct rdma_dev_addr *addr);

int rdma_addr_size(const struct sockaddr *addr);
int rdma_addr_size_in6(struct sockaddr_in6 *addr);
int rdma_addr_size_kss(struct __kernel_sockaddr_storage *addr);

static inline u16 ib_addr_get_pkey(struct rdma_dev_addr *dev_addr)
{
        return ((u16)dev_addr->broadcast[8] << 8) | (u16)dev_addr->broadcast[9];
}

static inline void ib_addr_set_pkey(struct rdma_dev_addr *dev_addr, u16 pkey)
{
        dev_addr->broadcast[8] = pkey >> 8;
        dev_addr->broadcast[9] = (unsigned char) pkey;
}

static inline void ib_addr_get_mgid(struct rdma_dev_addr *dev_addr,
                                    union ib_gid *gid)
{
        memcpy(gid, dev_addr->broadcast + 4, sizeof *gid);
}

static inline int rdma_addr_gid_offset(struct rdma_dev_addr *dev_addr)
{
        return dev_addr->dev_type == ARPHRD_INFINIBAND ? 4 : 0;
}

static inline u16 rdma_vlan_dev_vlan_id(const struct net_device *dev)
{
        return is_vlan_dev(dev) ? vlan_dev_vlan_id(dev) : 0xffff;
}

static inline int rdma_ip2gid(struct sockaddr *addr, union ib_gid *gid)
{
        switch (addr->sa_family) {
        case AF_INET:
                ipv6_addr_set_v4mapped(((struct sockaddr_in *)
                                        addr)->sin_addr.s_addr,
                                       (struct in6_addr *)gid);
                break;
        case AF_INET6:
                *(struct in6_addr *)&gid->raw =
                        ((struct sockaddr_in6 *)addr)->sin6_addr;
                break;
        default:
                return -EINVAL;
        }
        return 0;
}

/* Important - sockaddr should be a union of sockaddr_in and sockaddr_in6 */
static inline void rdma_gid2ip(struct sockaddr *out, const union ib_gid *gid)
{
        if (ipv6_addr_v4mapped((struct in6_addr *)gid)) {
                struct sockaddr_in *out_in = (struct sockaddr_in *)out;
                memset(out_in, 0, sizeof(*out_in));
                out_in->sin_family = AF_INET;
                memcpy(&out_in->sin_addr.s_addr, gid->raw + 12, 4);
        } else {
                struct sockaddr_in6 *out_in = (struct sockaddr_in6 *)out;
                memset(out_in, 0, sizeof(*out_in));
                out_in->sin6_family = AF_INET6;
                memcpy(&out_in->sin6_addr.s6_addr, gid->raw, 16);
        }
}

/*
 * rdma_get/set_sgid/dgid() APIs are applicable to IB, and iWarp.
 * They are not applicable to RoCE.
 * RoCE GIDs are derived from the IP addresses.
 */
static inline void rdma_addr_get_sgid(struct rdma_dev_addr *dev_addr, union ib_gid *gid)
{
        memcpy(gid, dev_addr->src_dev_addr + rdma_addr_gid_offset(dev_addr),
               sizeof(*gid));
}

static inline void rdma_addr_set_sgid(struct rdma_dev_addr *dev_addr, union ib_gid *gid)
{
        memcpy(dev_addr->src_dev_addr + rdma_addr_gid_offset(dev_addr), gid, sizeof *gid);
}

static inline void rdma_addr_get_dgid(struct rdma_dev_addr *dev_addr, union ib_gid *gid)
{
        memcpy(gid, dev_addr->dst_dev_addr + rdma_addr_gid_offset(dev_addr), sizeof *gid);
}

static inline void rdma_addr_set_dgid(struct rdma_dev_addr *dev_addr, union ib_gid *gid)
{
        memcpy(dev_addr->dst_dev_addr + rdma_addr_gid_offset(dev_addr), gid, sizeof *gid);
}

static inline enum ib_mtu iboe_get_mtu(int mtu)
{
        /*
         * Reduce IB headers from effective IBoE MTU.
         */
        mtu = mtu - (IB_GRH_BYTES + IB_UDP_BYTES + IB_BTH_BYTES +
                     IB_EXT_XRC_BYTES + IB_EXT_ATOMICETH_BYTES +
                     IB_ICRC_BYTES);

        if (mtu >= ib_mtu_enum_to_int(IB_MTU_4096))
                return IB_MTU_4096;
        else if (mtu >= ib_mtu_enum_to_int(IB_MTU_2048))
                return IB_MTU_2048;
        else if (mtu >= ib_mtu_enum_to_int(IB_MTU_1024))
                return IB_MTU_1024;
        else if (mtu >= ib_mtu_enum_to_int(IB_MTU_512))
                return IB_MTU_512;
        else if (mtu >= ib_mtu_enum_to_int(IB_MTU_256))
                return IB_MTU_256;
        else
                return 0;
}

static inline int rdma_link_local_addr(struct in6_addr *addr)
{
        if (addr->s6_addr32[0] == htonl(0xfe800000) &&
            addr->s6_addr32[1] == 0)
                return 1;

        return 0;
}

static inline void rdma_get_ll_mac(struct in6_addr *addr, u8 *mac)
{
        memcpy(mac, &addr->s6_addr[8], 3);
        memcpy(mac + 3, &addr->s6_addr[13], 3);
        mac[0] ^= 2;
}

static inline int rdma_is_multicast_addr(struct in6_addr *addr)
{
        __be32 ipv4_addr;

        if (addr->s6_addr[0] == 0xff)
                return 1;

        ipv4_addr = addr->s6_addr32[3];
        return (ipv6_addr_v4mapped(addr) && ipv4_is_multicast(ipv4_addr));
}

static inline void rdma_get_mcast_mac(struct in6_addr *addr, u8 *mac)
{
        int i;

        mac[0] = 0x33;
        mac[1] = 0x33;
        for (i = 2; i < 6; ++i)
                mac[i] = addr->s6_addr[i + 10];
}

static inline u16 rdma_get_vlan_id(union ib_gid *dgid)
{
        u16 vid;

        vid = dgid->raw[11] << 8 | dgid->raw[12];
        return vid < 0x1000 ? vid : 0xffff;
}

static inline struct net_device *rdma_vlan_dev_real_dev(const struct net_device *dev)
{
        return is_vlan_dev(dev) ? vlan_dev_real_dev(dev) : NULL;
}

#endif /* IB_ADDR_H */





















































    4 
    4 



















    4 
    4 
    4 

    4 

















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
/*
 * llc_core.c - Minimum needed routines for sap handling and module init/exit
 *
 * Copyright (c) 1997 by Procom Technology, Inc.
 *                  2001-2003 by Arnaldo Carvalho de Melo <acme@conectiva.com.br>
 *
 * This program can be redistributed or modified under the terms of the
 * GNU General Public License as published by the Free Software Foundation.
 * This program is distributed without any warranty or implied warranty
 * of merchantability or fitness for a particular purpose.
 *
 * See the GNU General Public License for more details.
 */

#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/if_ether.h>
#include <linux/netdevice.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/init.h>
#include <net/net_namespace.h>
#include <net/llc.h>

LIST_HEAD(llc_sap_list);
static DEFINE_SPINLOCK(llc_sap_list_lock);

/**
 *        llc_sap_alloc - allocates and initializes sap.
 *
 *        Allocates and initializes sap.
 */
static struct llc_sap *llc_sap_alloc(void)
{
        struct llc_sap *sap = kzalloc(sizeof(*sap), GFP_ATOMIC);
        int i;

        if (sap) {
                /* sap->laddr.mac - leave as a null, it's filled by bind */
                sap->state = LLC_SAP_STATE_ACTIVE;
                spin_lock_init(&sap->sk_lock);
                for (i = 0; i < LLC_SK_LADDR_HASH_ENTRIES; i++)
                        INIT_HLIST_NULLS_HEAD(&sap->sk_laddr_hash[i], i);
                refcount_set(&sap->refcnt, 1);
        }
        return sap;
}

static struct llc_sap *__llc_sap_find(unsigned char sap_value)
{
        struct llc_sap *sap;

        list_for_each_entry(sap, &llc_sap_list, node)
                if (sap->laddr.lsap == sap_value)
                        goto out;
        sap = NULL;
out:
        return sap;
}

/**
 *        llc_sap_find - searches a SAP in station
 *        @sap_value: sap to be found
 *
 *        Searches for a sap in the sap list of the LLC's station upon the sap ID.
 *        If the sap is found it will be refcounted and the user will have to do
 *        a llc_sap_put after use.
 *        Returns the sap or %NULL if not found.
 */
struct llc_sap *llc_sap_find(unsigned char sap_value)
{
        struct llc_sap *sap;

        rcu_read_lock_bh();
        sap = __llc_sap_find(sap_value);
        if (!sap || !llc_sap_hold_safe(sap))
                sap = NULL;
        rcu_read_unlock_bh();
        return sap;
}

/**
 *        llc_sap_open - open interface to the upper layers.
 *        @lsap: SAP number.
 *        @func: rcv func for datalink protos
 *
 *        Interface function to upper layer. Each one who wants to get a SAP
 *        (for example NetBEUI) should call this function. Returns the opened
 *        SAP for success, NULL for failure.
 */
struct llc_sap *llc_sap_open(unsigned char lsap,
                             int (*func)(struct sk_buff *skb,
                                         struct net_device *dev,
                                         struct packet_type *pt,
                                         struct net_device *orig_dev))
{
        struct llc_sap *sap = NULL;

        spin_lock_bh(&llc_sap_list_lock);
        if (__llc_sap_find(lsap)) /* SAP already exists */
                goto out;
        sap = llc_sap_alloc();
        if (!sap)
                goto out;
        sap->laddr.lsap = lsap;
        sap->rcv_func        = func;
        list_add_tail_rcu(&sap->node, &llc_sap_list);
out:
        spin_unlock_bh(&llc_sap_list_lock);
        return sap;
}

/**
 *        llc_sap_close - close interface for upper layers.
 *        @sap: SAP to be closed.
 *
 *        Close interface function to upper layer. Each one who wants to
 *        close an open SAP (for example NetBEUI) should call this function.
 *         Removes this sap from the list of saps in the station and then
 *         frees the memory for this sap.
 */
void llc_sap_close(struct llc_sap *sap)
{
        WARN_ON(sap->sk_count);

        spin_lock_bh(&llc_sap_list_lock);
        list_del_rcu(&sap->node);
        spin_unlock_bh(&llc_sap_list_lock);

        kfree_rcu(sap, rcu);
}

static struct packet_type llc_packet_type __read_mostly = {
        .type = cpu_to_be16(ETH_P_802_2),
        .func = llc_rcv,
};

static int __init llc_init(void)
{
        dev_add_pack(&llc_packet_type);
        return 0;
}

static void __exit llc_exit(void)
{
        dev_remove_pack(&llc_packet_type);
}

module_init(llc_init);
module_exit(llc_exit);

EXPORT_SYMBOL(llc_sap_list);
EXPORT_SYMBOL(llc_sap_find);
EXPORT_SYMBOL(llc_sap_open);
EXPORT_SYMBOL(llc_sap_close);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Procom 1997, Jay Schullist 2001, Arnaldo C. Melo 2001-2003");
MODULE_DESCRIPTION("LLC IEEE 802.2 core support");







































































































































































































   35 


   35 

   35 




   35 
   35 









   35 

   35 













   35 


























   34 


   35 



   35 













































































    8 

    8 

    8 




















    8 

    8 





    8 
    8 


    8 

    8 


    8 


    8 


    8 


    8 






    8 
    8 









    8 






    8 



    8 


    8 









    8 






































































































































































































































































































































































































































































































    8 






    8 




    8 































































   36 

   36 


   36 
   36 




   35 






   35 
   35 











    6 




    6 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2002-2005, Instant802 Networks, Inc.
 * Copyright 2005-2006, Devicescape Software, Inc.
 * Copyright (c) 2006 Jiri Benc <jbenc@suse.cz>
 * Copyright 2017        Intel Deutschland GmbH
 * Copyright (C) 2019, 2022-2024 Intel Corporation
 */

#include <linux/kernel.h>
#include <linux/rtnetlink.h>
#include <linux/module.h>
#include <linux/slab.h>
#include "rate.h"
#include "ieee80211_i.h"
#include "debugfs.h"

struct rate_control_alg {
        struct list_head list;
        const struct rate_control_ops *ops;
};

static LIST_HEAD(rate_ctrl_algs);
static DEFINE_MUTEX(rate_ctrl_mutex);

static char *ieee80211_default_rc_algo = CONFIG_MAC80211_RC_DEFAULT;
module_param(ieee80211_default_rc_algo, charp, 0644);
MODULE_PARM_DESC(ieee80211_default_rc_algo,
                 "Default rate control algorithm for mac80211 to use");

void rate_control_rate_init(struct link_sta_info *link_sta)
{
        struct sta_info *sta = link_sta->sta;
        struct ieee80211_local *local = sta->sdata->local;
        struct rate_control_ref *ref = sta->rate_ctrl;
        struct ieee80211_sta *ista = &sta->sta;
        void *priv_sta = sta->rate_ctrl_priv;
        struct ieee80211_supported_band *sband;
        struct ieee80211_chanctx_conf *chanctx_conf;

        ieee80211_sta_init_nss(link_sta);

        if (!ref)
                return;

        /* SW rate control isn't supported with MLO right now */
        if (WARN_ON(ieee80211_vif_is_mld(&sta->sdata->vif)))
                return;

        rcu_read_lock();

        chanctx_conf = rcu_dereference(sta->sdata->vif.bss_conf.chanctx_conf);
        if (WARN_ON(!chanctx_conf)) {
                rcu_read_unlock();
                return;
        }

        sband = local->hw.wiphy->bands[chanctx_conf->def.chan->band];

        /* TODO: check for minstrel_s1g ? */
        if (sband->band == NL80211_BAND_S1GHZ) {
                ieee80211_s1g_sta_rate_init(sta);
                rcu_read_unlock();
                return;
        }

        spin_lock_bh(&sta->rate_ctrl_lock);
        ref->ops->rate_init(ref->priv, sband, &chanctx_conf->def, ista,
                            priv_sta);
        spin_unlock_bh(&sta->rate_ctrl_lock);
        rcu_read_unlock();
        set_sta_flag(sta, WLAN_STA_RATE_CONTROL);
}

void rate_control_rate_init_all_links(struct sta_info *sta)
{
        int link_id;

        for (link_id = 0; link_id < ARRAY_SIZE(sta->link); link_id++) {
                struct link_sta_info *link_sta;

                link_sta = sdata_dereference(sta->link[link_id], sta->sdata);
                if (!link_sta)
                        continue;

                rate_control_rate_init(link_sta);
        }
}

void rate_control_tx_status(struct ieee80211_local *local,
                            struct ieee80211_tx_status *st)
{
        struct rate_control_ref *ref = local->rate_ctrl;
        struct sta_info *sta = container_of(st->sta, struct sta_info, sta);
        void *priv_sta = sta->rate_ctrl_priv;
        struct ieee80211_supported_band *sband;

        if (!ref || !test_sta_flag(sta, WLAN_STA_RATE_CONTROL))
                return;

        sband = local->hw.wiphy->bands[st->info->band];

        spin_lock_bh(&sta->rate_ctrl_lock);
        if (ref->ops->tx_status_ext)
                ref->ops->tx_status_ext(ref->priv, sband, priv_sta, st);
        else if (st->skb)
                ref->ops->tx_status(ref->priv, sband, st->sta, priv_sta, st->skb);
        else
                WARN_ON_ONCE(1);

        spin_unlock_bh(&sta->rate_ctrl_lock);
}

void rate_control_rate_update(struct ieee80211_local *local,
                              struct ieee80211_supported_band *sband,
                              struct link_sta_info *link_sta,
                              u32 changed)
{
        struct rate_control_ref *ref = local->rate_ctrl;
        struct sta_info *sta = link_sta->sta;
        struct ieee80211_sta *ista = &sta->sta;
        void *priv_sta = sta->rate_ctrl_priv;
        struct ieee80211_chanctx_conf *chanctx_conf;

        if (ref && ref->ops->rate_update) {
                rcu_read_lock();

                chanctx_conf = rcu_dereference(sta->sdata->vif.bss_conf.chanctx_conf);
                if (WARN_ON(!chanctx_conf)) {
                        rcu_read_unlock();
                        return;
                }

                spin_lock_bh(&sta->rate_ctrl_lock);
                ref->ops->rate_update(ref->priv, sband, &chanctx_conf->def,
                                      ista, priv_sta, changed);
                spin_unlock_bh(&sta->rate_ctrl_lock);
                rcu_read_unlock();
        }

        if (sta->uploaded)
                drv_link_sta_rc_update(local, sta->sdata, link_sta->pub,
                                       changed);
}

int ieee80211_rate_control_register(const struct rate_control_ops *ops)
{
        struct rate_control_alg *alg;

        if (!ops->name)
                return -EINVAL;

        mutex_lock(&rate_ctrl_mutex);
        list_for_each_entry(alg, &rate_ctrl_algs, list) {
                if (!strcmp(alg->ops->name, ops->name)) {
                        /* don't register an algorithm twice */
                        WARN_ON(1);
                        mutex_unlock(&rate_ctrl_mutex);
                        return -EALREADY;
                }
        }

        alg = kzalloc(sizeof(*alg), GFP_KERNEL);
        if (alg == NULL) {
                mutex_unlock(&rate_ctrl_mutex);
                return -ENOMEM;
        }
        alg->ops = ops;

        list_add_tail(&alg->list, &rate_ctrl_algs);
        mutex_unlock(&rate_ctrl_mutex);

        return 0;
}
EXPORT_SYMBOL(ieee80211_rate_control_register);

void ieee80211_rate_control_unregister(const struct rate_control_ops *ops)
{
        struct rate_control_alg *alg;

        mutex_lock(&rate_ctrl_mutex);
        list_for_each_entry(alg, &rate_ctrl_algs, list) {
                if (alg->ops == ops) {
                        list_del(&alg->list);
                        kfree(alg);
                        break;
                }
        }
        mutex_unlock(&rate_ctrl_mutex);
}
EXPORT_SYMBOL(ieee80211_rate_control_unregister);

static const struct rate_control_ops *
ieee80211_try_rate_control_ops_get(const char *name)
{
        struct rate_control_alg *alg;
        const struct rate_control_ops *ops = NULL;

        if (!name)
                return NULL;

        mutex_lock(&rate_ctrl_mutex);
        list_for_each_entry(alg, &rate_ctrl_algs, list) {
                if (!strcmp(alg->ops->name, name)) {
                        ops = alg->ops;
                        break;
                }
        }
        mutex_unlock(&rate_ctrl_mutex);
        return ops;
}

/* Get the rate control algorithm. */
static const struct rate_control_ops *
ieee80211_rate_control_ops_get(const char *name)
{
        const struct rate_control_ops *ops;
        const char *alg_name;

        kernel_param_lock(THIS_MODULE);
        if (!name)
                alg_name = ieee80211_default_rc_algo;
        else
                alg_name = name;

        ops = ieee80211_try_rate_control_ops_get(alg_name);
        if (!ops && name)
                /* try default if specific alg requested but not found */
                ops = ieee80211_try_rate_control_ops_get(ieee80211_default_rc_algo);

        /* Note: check for > 0 is intentional to avoid clang warning */
        if (!ops && (strlen(CONFIG_MAC80211_RC_DEFAULT) > 0))
                /* try built-in one if specific alg requested but not found */
                ops = ieee80211_try_rate_control_ops_get(CONFIG_MAC80211_RC_DEFAULT);

        kernel_param_unlock(THIS_MODULE);

        return ops;
}

#ifdef CONFIG_MAC80211_DEBUGFS
static ssize_t rcname_read(struct file *file, char __user *userbuf,
                           size_t count, loff_t *ppos)
{
        struct rate_control_ref *ref = file->private_data;
        int len = strlen(ref->ops->name);

        return simple_read_from_buffer(userbuf, count, ppos,
                                       ref->ops->name, len);
}

const struct debugfs_short_fops rcname_ops = {
        .read = rcname_read,
        .llseek = default_llseek,
};
#endif

static struct rate_control_ref *
rate_control_alloc(const char *name, struct ieee80211_local *local)
{
        struct rate_control_ref *ref;

        ref = kmalloc(sizeof(struct rate_control_ref), GFP_KERNEL);
        if (!ref)
                return NULL;
        ref->ops = ieee80211_rate_control_ops_get(name);
        if (!ref->ops)
                goto free;

        ref->priv = ref->ops->alloc(&local->hw);
        if (!ref->priv)
                goto free;
        return ref;

free:
        kfree(ref);
        return NULL;
}

static void rate_control_free(struct ieee80211_local *local,
                              struct rate_control_ref *ctrl_ref)
{
        ctrl_ref->ops->free(ctrl_ref->priv);

#ifdef CONFIG_MAC80211_DEBUGFS
        debugfs_remove_recursive(local->debugfs.rcdir);
        local->debugfs.rcdir = NULL;
#endif

        kfree(ctrl_ref);
}

void ieee80211_check_rate_mask(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_supported_band *sband;
        u32 user_mask, basic_rates = link->conf->basic_rates;
        enum nl80211_band band;

        if (WARN_ON(!link->conf->chanreq.oper.chan))
                return;

        band = link->conf->chanreq.oper.chan->band;
        if (band == NL80211_BAND_S1GHZ) {
                /* TODO */
                return;
        }

        if (WARN_ON_ONCE(!basic_rates))
                return;

        user_mask = sdata->rc_rateidx_mask[band];
        sband = local->hw.wiphy->bands[band];

        if (user_mask & basic_rates)
                return;

        sdata_dbg(sdata,
                  "no overlap between basic rates (0x%x) and user mask (0x%x on band %d) - clearing the latter",
                  basic_rates, user_mask, band);
        sdata->rc_rateidx_mask[band] = (1 << sband->n_bitrates) - 1;
}

static bool rc_no_data_or_no_ack_use_min(struct ieee80211_tx_rate_control *txrc)
{
        struct sk_buff *skb = txrc->skb;
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);

        return (info->flags & (IEEE80211_TX_CTL_NO_ACK |
                               IEEE80211_TX_CTL_USE_MINRATE)) ||
                !ieee80211_is_tx_data(skb);
}

static void rc_send_low_basicrate(struct ieee80211_tx_rate *rate,
                                  u32 basic_rates,
                                  struct ieee80211_supported_band *sband)
{
        u8 i;

        if (sband->band == NL80211_BAND_S1GHZ) {
                /* TODO */
                rate->flags |= IEEE80211_TX_RC_S1G_MCS;
                rate->idx = 0;
                return;
        }

        if (basic_rates == 0)
                return; /* assume basic rates unknown and accept rate */
        if (rate->idx < 0)
                return;
        if (basic_rates & (1 << rate->idx))
                return; /* selected rate is a basic rate */

        for (i = rate->idx + 1; i <= sband->n_bitrates; i++) {
                if (basic_rates & (1 << i)) {
                        rate->idx = i;
                        return;
                }
        }

        /* could not find a basic rate; use original selection */
}

static void __rate_control_send_low(struct ieee80211_hw *hw,
                                    struct ieee80211_supported_band *sband,
                                    struct ieee80211_sta *sta,
                                    struct ieee80211_tx_info *info,
                                    u32 rate_mask)
{
        int i;
        u32 rate_flags =
                ieee80211_chandef_rate_flags(&hw->conf.chandef);

        if (sband->band == NL80211_BAND_S1GHZ) {
                info->control.rates[0].flags |= IEEE80211_TX_RC_S1G_MCS;
                info->control.rates[0].idx = 0;
                return;
        }

        if ((sband->band == NL80211_BAND_2GHZ) &&
            (info->flags & IEEE80211_TX_CTL_NO_CCK_RATE))
                rate_flags |= IEEE80211_RATE_ERP_G;

        info->control.rates[0].idx = 0;
        for (i = 0; i < sband->n_bitrates; i++) {
                if (!(rate_mask & BIT(i)))
                        continue;

                if ((rate_flags & sband->bitrates[i].flags) != rate_flags)
                        continue;

                if (!rate_supported(sta, sband->band, i))
                        continue;

                info->control.rates[0].idx = i;
                break;
        }
        WARN_ONCE(i == sband->n_bitrates,
                  "no supported rates for sta %pM (0x%x, band %d) in rate_mask 0x%x with flags 0x%x\n",
                  sta ? sta->addr : NULL,
                  sta ? sta->deflink.supp_rates[sband->band] : -1,
                  sband->band,
                  rate_mask, rate_flags);

        info->control.rates[0].count =
                (info->flags & IEEE80211_TX_CTL_NO_ACK) ?
                1 : hw->max_rate_tries;

        info->control.skip_table = 1;
}


static bool rate_control_send_low(struct ieee80211_sta *pubsta,
                                  struct ieee80211_tx_rate_control *txrc)
{
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(txrc->skb);
        struct ieee80211_supported_band *sband = txrc->sband;
        struct sta_info *sta;
        int mcast_rate;
        bool use_basicrate = false;

        if (!pubsta || rc_no_data_or_no_ack_use_min(txrc)) {
                __rate_control_send_low(txrc->hw, sband, pubsta, info,
                                        txrc->rate_idx_mask);

                if (!pubsta && txrc->bss) {
                        mcast_rate = txrc->bss_conf->mcast_rate[sband->band];
                        if (mcast_rate > 0) {
                                info->control.rates[0].idx = mcast_rate - 1;
                                return true;
                        }
                        use_basicrate = true;
                } else if (pubsta) {
                        sta = container_of(pubsta, struct sta_info, sta);
                        if (ieee80211_vif_is_mesh(&sta->sdata->vif))
                                use_basicrate = true;
                }

                if (use_basicrate)
                        rc_send_low_basicrate(&info->control.rates[0],
                                              txrc->bss_conf->basic_rates,
                                              sband);

                return true;
        }
        return false;
}

static bool rate_idx_match_legacy_mask(s8 *rate_idx, int n_bitrates, u32 mask)
{
        int j;

        /* See whether the selected rate or anything below it is allowed. */
        for (j = *rate_idx; j >= 0; j--) {
                if (mask & (1 << j)) {
                        /* Okay, found a suitable rate. Use it. */
                        *rate_idx = j;
                        return true;
                }
        }

        /* Try to find a higher rate that would be allowed */
        for (j = *rate_idx + 1; j < n_bitrates; j++) {
                if (mask & (1 << j)) {
                        /* Okay, found a suitable rate. Use it. */
                        *rate_idx = j;
                        return true;
                }
        }
        return false;
}

static bool rate_idx_match_mcs_mask(s8 *rate_idx, u8 *mcs_mask)
{
        int i, j;
        int ridx, rbit;

        ridx = *rate_idx / 8;
        rbit = *rate_idx % 8;

        /* sanity check */
        if (ridx < 0 || ridx >= IEEE80211_HT_MCS_MASK_LEN)
                return false;

        /* See whether the selected rate or anything below it is allowed. */
        for (i = ridx; i >= 0; i--) {
                for (j = rbit; j >= 0; j--)
                        if (mcs_mask[i] & BIT(j)) {
                                *rate_idx = i * 8 + j;
                                return true;
                        }
                rbit = 7;
        }

        /* Try to find a higher rate that would be allowed */
        ridx = (*rate_idx + 1) / 8;
        rbit = (*rate_idx + 1) % 8;

        for (i = ridx; i < IEEE80211_HT_MCS_MASK_LEN; i++) {
                for (j = rbit; j < 8; j++)
                        if (mcs_mask[i] & BIT(j)) {
                                *rate_idx = i * 8 + j;
                                return true;
                        }
                rbit = 0;
        }
        return false;
}

static bool rate_idx_match_vht_mcs_mask(s8 *rate_idx, u16 *vht_mask)
{
        int i, j;
        int ridx, rbit;

        ridx = *rate_idx >> 4;
        rbit = *rate_idx & 0xf;

        if (ridx < 0 || ridx >= NL80211_VHT_NSS_MAX)
                return false;

        /* See whether the selected rate or anything below it is allowed. */
        for (i = ridx; i >= 0; i--) {
                for (j = rbit; j >= 0; j--) {
                        if (vht_mask[i] & BIT(j)) {
                                *rate_idx = (i << 4) | j;
                                return true;
                        }
                }
                rbit = 15;
        }

        /* Try to find a higher rate that would be allowed */
        ridx = (*rate_idx + 1) >> 4;
        rbit = (*rate_idx + 1) & 0xf;

        for (i = ridx; i < NL80211_VHT_NSS_MAX; i++) {
                for (j = rbit; j < 16; j++) {
                        if (vht_mask[i] & BIT(j)) {
                                *rate_idx = (i << 4) | j;
                                return true;
                        }
                }
                rbit = 0;
        }
        return false;
}

static void rate_idx_match_mask(s8 *rate_idx, u16 *rate_flags,
                                struct ieee80211_supported_band *sband,
                                enum nl80211_chan_width chan_width,
                                u32 mask,
                                u8 mcs_mask[IEEE80211_HT_MCS_MASK_LEN],
                                u16 vht_mask[NL80211_VHT_NSS_MAX])
{
        if (*rate_flags & IEEE80211_TX_RC_VHT_MCS) {
                /* handle VHT rates */
                if (rate_idx_match_vht_mcs_mask(rate_idx, vht_mask))
                        return;

                *rate_idx = 0;
                /* keep protection flags */
                *rate_flags &= (IEEE80211_TX_RC_USE_RTS_CTS |
                                IEEE80211_TX_RC_USE_CTS_PROTECT |
                                IEEE80211_TX_RC_USE_SHORT_PREAMBLE);

                *rate_flags |= IEEE80211_TX_RC_MCS;
                if (chan_width == NL80211_CHAN_WIDTH_40)
                        *rate_flags |= IEEE80211_TX_RC_40_MHZ_WIDTH;

                if (rate_idx_match_mcs_mask(rate_idx, mcs_mask))
                        return;

                /* also try the legacy rates. */
                *rate_flags &= ~(IEEE80211_TX_RC_MCS |
                                 IEEE80211_TX_RC_40_MHZ_WIDTH);
                if (rate_idx_match_legacy_mask(rate_idx, sband->n_bitrates,
                                               mask))
                        return;
        } else if (*rate_flags & IEEE80211_TX_RC_MCS) {
                /* handle HT rates */
                if (rate_idx_match_mcs_mask(rate_idx, mcs_mask))
                        return;

                /* also try the legacy rates. */
                *rate_idx = 0;
                /* keep protection flags */
                *rate_flags &= (IEEE80211_TX_RC_USE_RTS_CTS |
                                IEEE80211_TX_RC_USE_CTS_PROTECT |
                                IEEE80211_TX_RC_USE_SHORT_PREAMBLE);
                if (rate_idx_match_legacy_mask(rate_idx, sband->n_bitrates,
                                               mask))
                        return;
        } else {
                /* handle legacy rates */
                if (rate_idx_match_legacy_mask(rate_idx, sband->n_bitrates,
                                               mask))
                        return;

                /* if HT BSS, and we handle a data frame, also try HT rates */
                switch (chan_width) {
                case NL80211_CHAN_WIDTH_20_NOHT:
                case NL80211_CHAN_WIDTH_5:
                case NL80211_CHAN_WIDTH_10:
                        return;
                default:
                        break;
                }

                *rate_idx = 0;
                /* keep protection flags */
                *rate_flags &= (IEEE80211_TX_RC_USE_RTS_CTS |
                                IEEE80211_TX_RC_USE_CTS_PROTECT |
                                IEEE80211_TX_RC_USE_SHORT_PREAMBLE);

                *rate_flags |= IEEE80211_TX_RC_MCS;

                if (chan_width == NL80211_CHAN_WIDTH_40)
                        *rate_flags |= IEEE80211_TX_RC_40_MHZ_WIDTH;

                if (rate_idx_match_mcs_mask(rate_idx, mcs_mask))
                        return;
        }

        /*
         * Uh.. No suitable rate exists. This should not really happen with
         * sane TX rate mask configurations. However, should someone manage to
         * configure supported rates and TX rate mask in incompatible way,
         * allow the frame to be transmitted with whatever the rate control
         * selected.
         */
}

static void rate_fixup_ratelist(struct ieee80211_vif *vif,
                                struct ieee80211_supported_band *sband,
                                struct ieee80211_tx_info *info,
                                struct ieee80211_tx_rate *rates,
                                int max_rates)
{
        struct ieee80211_rate *rate;
        bool inval = false;
        int i;

        /*
         * Set up the RTS/CTS rate as the fastest basic rate
         * that is not faster than the data rate unless there
         * is no basic rate slower than the data rate, in which
         * case we pick the slowest basic rate
         *
         * XXX: Should this check all retry rates?
         */
        if (!(rates[0].flags &
              (IEEE80211_TX_RC_MCS | IEEE80211_TX_RC_VHT_MCS))) {
                u32 basic_rates = vif->bss_conf.basic_rates;
                s8 baserate = basic_rates ? ffs(basic_rates) - 1 : 0;

                rate = &sband->bitrates[rates[0].idx];

                for (i = 0; i < sband->n_bitrates; i++) {
                        /* must be a basic rate */
                        if (!(basic_rates & BIT(i)))
                                continue;
                        /* must not be faster than the data rate */
                        if (sband->bitrates[i].bitrate > rate->bitrate)
                                continue;
                        /* maximum */
                        if (sband->bitrates[baserate].bitrate <
                             sband->bitrates[i].bitrate)
                                baserate = i;
                }

                info->control.rts_cts_rate_idx = baserate;
        }

        for (i = 0; i < max_rates; i++) {
                /*
                 * make sure there's no valid rate following
                 * an invalid one, just in case drivers don't
                 * take the API seriously to stop at -1.
                 */
                if (inval) {
                        rates[i].idx = -1;
                        continue;
                }
                if (rates[i].idx < 0) {
                        inval = true;
                        continue;
                }

                /*
                 * For now assume MCS is already set up correctly, this
                 * needs to be fixed.
                 */
                if (rates[i].flags & IEEE80211_TX_RC_MCS) {
                        WARN_ON(rates[i].idx > 76);

                        if (!(rates[i].flags & IEEE80211_TX_RC_USE_RTS_CTS) &&
                            info->control.use_cts_prot)
                                rates[i].flags |=
                                        IEEE80211_TX_RC_USE_CTS_PROTECT;
                        continue;
                }

                if (rates[i].flags & IEEE80211_TX_RC_VHT_MCS) {
                        WARN_ON(ieee80211_rate_get_vht_mcs(&rates[i]) > 9);
                        continue;
                }

                /* set up RTS protection if desired */
                if (info->control.use_rts) {
                        rates[i].flags |= IEEE80211_TX_RC_USE_RTS_CTS;
                        info->control.use_cts_prot = false;
                }

                /* RC is busted */
                if (WARN_ON_ONCE(rates[i].idx >= sband->n_bitrates)) {
                        rates[i].idx = -1;
                        continue;
                }

                rate = &sband->bitrates[rates[i].idx];

                /* set up short preamble */
                if (info->control.short_preamble &&
                    rate->flags & IEEE80211_RATE_SHORT_PREAMBLE)
                        rates[i].flags |= IEEE80211_TX_RC_USE_SHORT_PREAMBLE;

                /* set up G protection */
                if (!(rates[i].flags & IEEE80211_TX_RC_USE_RTS_CTS) &&
                    info->control.use_cts_prot &&
                    rate->flags & IEEE80211_RATE_ERP_G)
                        rates[i].flags |= IEEE80211_TX_RC_USE_CTS_PROTECT;
        }
}


static void rate_control_fill_sta_table(struct ieee80211_sta *sta,
                                        struct ieee80211_tx_info *info,
                                        struct ieee80211_tx_rate *rates,
                                        int max_rates)
{
        struct ieee80211_sta_rates *ratetbl = NULL;
        int i;

        if (sta && !info->control.skip_table)
                ratetbl = rcu_dereference(sta->rates);

        /* Fill remaining rate slots with data from the sta rate table. */
        max_rates = min_t(int, max_rates, IEEE80211_TX_RATE_TABLE_SIZE);
        for (i = 0; i < max_rates; i++) {
                if (i < ARRAY_SIZE(info->control.rates) &&
                    info->control.rates[i].idx >= 0 &&
                    info->control.rates[i].count) {
                        if (rates != info->control.rates)
                                rates[i] = info->control.rates[i];
                } else if (ratetbl) {
                        rates[i].idx = ratetbl->rate[i].idx;
                        rates[i].flags = ratetbl->rate[i].flags;
                        if (info->control.use_rts)
                                rates[i].count = ratetbl->rate[i].count_rts;
                        else if (info->control.use_cts_prot)
                                rates[i].count = ratetbl->rate[i].count_cts;
                        else
                                rates[i].count = ratetbl->rate[i].count;
                } else {
                        rates[i].idx = -1;
                        rates[i].count = 0;
                }

                if (rates[i].idx < 0 || !rates[i].count)
                        break;
        }
}

static bool rate_control_cap_mask(struct ieee80211_sub_if_data *sdata,
                                  struct ieee80211_supported_band *sband,
                                  struct ieee80211_sta *sta, u32 *mask,
                                  u8 mcs_mask[IEEE80211_HT_MCS_MASK_LEN],
                                  u16 vht_mask[NL80211_VHT_NSS_MAX])
{
        u32 i, flags;

        *mask = sdata->rc_rateidx_mask[sband->band];
        flags = ieee80211_chandef_rate_flags(&sdata->vif.bss_conf.chanreq.oper);
        for (i = 0; i < sband->n_bitrates; i++) {
                if ((flags & sband->bitrates[i].flags) != flags)
                        *mask &= ~BIT(i);
        }

        if (*mask == (1 << sband->n_bitrates) - 1 &&
            !sdata->rc_has_mcs_mask[sband->band] &&
            !sdata->rc_has_vht_mcs_mask[sband->band])
                return false;

        if (sdata->rc_has_mcs_mask[sband->band])
                memcpy(mcs_mask, sdata->rc_rateidx_mcs_mask[sband->band],
                       IEEE80211_HT_MCS_MASK_LEN);
        else
                memset(mcs_mask, 0xff, IEEE80211_HT_MCS_MASK_LEN);

        if (sdata->rc_has_vht_mcs_mask[sband->band])
                memcpy(vht_mask, sdata->rc_rateidx_vht_mcs_mask[sband->band],
                       sizeof(u16) * NL80211_VHT_NSS_MAX);
        else
                memset(vht_mask, 0xff, sizeof(u16) * NL80211_VHT_NSS_MAX);

        if (sta) {
                __le16 sta_vht_cap;
                u16 sta_vht_mask[NL80211_VHT_NSS_MAX];

                /* Filter out rates that the STA does not support */
                *mask &= sta->deflink.supp_rates[sband->band];
                for (i = 0; i < IEEE80211_HT_MCS_MASK_LEN; i++)
                        mcs_mask[i] &= sta->deflink.ht_cap.mcs.rx_mask[i];

                sta_vht_cap = sta->deflink.vht_cap.vht_mcs.rx_mcs_map;
                ieee80211_get_vht_mask_from_cap(sta_vht_cap, sta_vht_mask);
                for (i = 0; i < NL80211_VHT_NSS_MAX; i++)
                        vht_mask[i] &= sta_vht_mask[i];
        }

        return true;
}

static void
rate_control_apply_mask_ratetbl(struct sta_info *sta,
                                struct ieee80211_supported_band *sband,
                                struct ieee80211_sta_rates *rates)
{
        int i;
        u32 mask;
        u8 mcs_mask[IEEE80211_HT_MCS_MASK_LEN];
        u16 vht_mask[NL80211_VHT_NSS_MAX];
        enum nl80211_chan_width chan_width;

        if (!rate_control_cap_mask(sta->sdata, sband, &sta->sta, &mask,
                                   mcs_mask, vht_mask))
                return;

        chan_width = sta->sdata->vif.bss_conf.chanreq.oper.width;
        for (i = 0; i < IEEE80211_TX_RATE_TABLE_SIZE; i++) {
                if (rates->rate[i].idx < 0)
                        break;

                rate_idx_match_mask(&rates->rate[i].idx, &rates->rate[i].flags,
                                    sband, chan_width, mask, mcs_mask,
                                    vht_mask);
        }
}

static void rate_control_apply_mask(struct ieee80211_sub_if_data *sdata,
                                    struct ieee80211_sta *sta,
                                    struct ieee80211_supported_band *sband,
                                    struct ieee80211_tx_rate *rates,
                                    int max_rates)
{
        enum nl80211_chan_width chan_width;
        u8 mcs_mask[IEEE80211_HT_MCS_MASK_LEN];
        u32 mask;
        u16 rate_flags, vht_mask[NL80211_VHT_NSS_MAX];
        int i;

        /*
         * Try to enforce the rateidx mask the user wanted. skip this if the
         * default mask (allow all rates) is used to save some processing for
         * the common case.
         */
        if (!rate_control_cap_mask(sdata, sband, sta, &mask, mcs_mask,
                                   vht_mask))
                return;

        /*
         * Make sure the rate index selected for each TX rate is
         * included in the configured mask and change the rate indexes
         * if needed.
         */
        chan_width = sdata->vif.bss_conf.chanreq.oper.width;
        for (i = 0; i < max_rates; i++) {
                /* Skip invalid rates */
                if (rates[i].idx < 0)
                        break;

                rate_flags = rates[i].flags;
                rate_idx_match_mask(&rates[i].idx, &rate_flags, sband,
                                    chan_width, mask, mcs_mask, vht_mask);
                rates[i].flags = rate_flags;
        }
}

void ieee80211_get_tx_rates(struct ieee80211_vif *vif,
                            struct ieee80211_sta *sta,
                            struct sk_buff *skb,
                            struct ieee80211_tx_rate *dest,
                            int max_rates)
{
        struct ieee80211_sub_if_data *sdata;
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
        struct ieee80211_supported_band *sband;
        u32 mask = ~0;

        rate_control_fill_sta_table(sta, info, dest, max_rates);

        if (!vif)
                return;

        sdata = vif_to_sdata(vif);
        sband = sdata->local->hw.wiphy->bands[info->band];

        if (ieee80211_is_tx_data(skb))
                rate_control_apply_mask(sdata, sta, sband, dest, max_rates);

        if (!(info->control.flags & IEEE80211_TX_CTRL_DONT_USE_RATE_MASK))
                mask = sdata->rc_rateidx_mask[info->band];

        if (dest[0].idx < 0)
                __rate_control_send_low(&sdata->local->hw, sband, sta, info,
                                        mask);

        if (sta)
                rate_fixup_ratelist(vif, sband, info, dest, max_rates);
}
EXPORT_SYMBOL(ieee80211_get_tx_rates);

void rate_control_get_rate(struct ieee80211_sub_if_data *sdata,
                           struct sta_info *sta,
                           struct ieee80211_tx_rate_control *txrc)
{
        struct rate_control_ref *ref = sdata->local->rate_ctrl;
        void *priv_sta = NULL;
        struct ieee80211_sta *ista = NULL;
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(txrc->skb);
        int i;

        for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) {
                info->control.rates[i].idx = -1;
                info->control.rates[i].flags = 0;
                info->control.rates[i].count = 0;
        }

        if (rate_control_send_low(sta ? &sta->sta : NULL, txrc))
                return;

        if (ieee80211_hw_check(&sdata->local->hw, HAS_RATE_CONTROL))
                return;

        if (sta && test_sta_flag(sta, WLAN_STA_RATE_CONTROL)) {
                ista = &sta->sta;
                priv_sta = sta->rate_ctrl_priv;
        }

        if (ista) {
                spin_lock_bh(&sta->rate_ctrl_lock);
                ref->ops->get_rate(ref->priv, ista, priv_sta, txrc);
                spin_unlock_bh(&sta->rate_ctrl_lock);
        } else {
                rate_control_send_low(NULL, txrc);
        }

        if (ieee80211_hw_check(&sdata->local->hw, SUPPORTS_RC_TABLE))
                return;

        ieee80211_get_tx_rates(&sdata->vif, ista, txrc->skb,
                               info->control.rates,
                               ARRAY_SIZE(info->control.rates));
}

int rate_control_set_rates(struct ieee80211_hw *hw,
                           struct ieee80211_sta *pubsta,
                           struct ieee80211_sta_rates *rates)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
        struct ieee80211_sta_rates *old;
        struct ieee80211_supported_band *sband;

        sband = ieee80211_get_sband(sta->sdata);
        if (!sband)
                return -EINVAL;
        rate_control_apply_mask_ratetbl(sta, sband, rates);
        /*
         * mac80211 guarantees that this function will not be called
         * concurrently, so the following RCU access is safe, even without
         * extra locking. This can not be checked easily, so we just set
         * the condition to true.
         */
        old = rcu_dereference_protected(pubsta->rates, true);
        rcu_assign_pointer(pubsta->rates, rates);
        if (old)
                kfree_rcu(old, rcu_head);

        if (sta->uploaded)
                drv_sta_rate_tbl_update(hw_to_local(hw), sta->sdata, pubsta);

        ieee80211_sta_set_expected_throughput(pubsta, sta_get_expected_throughput(sta));

        return 0;
}
EXPORT_SYMBOL(rate_control_set_rates);

int ieee80211_init_rate_ctrl_alg(struct ieee80211_local *local,
                                 const char *name)
{
        struct rate_control_ref *ref;

        ASSERT_RTNL();

        if (local->open_count)
                return -EBUSY;

        if (ieee80211_hw_check(&local->hw, HAS_RATE_CONTROL)) {
                if (WARN_ON(!local->ops->set_rts_threshold))
                        return -EINVAL;
                return 0;
        }

        ref = rate_control_alloc(name, local);
        if (!ref) {
                wiphy_warn(local->hw.wiphy,
                           "Failed to select rate control algorithm\n");
                return -ENOENT;
        }

        WARN_ON(local->rate_ctrl);
        local->rate_ctrl = ref;

        wiphy_debug(local->hw.wiphy, "Selected rate control algorithm '%s'\n",
                    ref->ops->name);

        return 0;
}

void rate_control_deinitialize(struct ieee80211_local *local)
{
        struct rate_control_ref *ref;

        ref = local->rate_ctrl;

        if (!ref)
                return;

        local->rate_ctrl = NULL;
        rate_control_free(local, ref);
}











































































































































































































































    2 




























    2 































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * PTP virtual clock driver
 *
 * Copyright 2021 NXP
 */
#include <linux/slab.h>
#include <linux/hashtable.h>
#include "ptp_private.h"

#define PTP_VCLOCK_CC_SHIFT                31
#define PTP_VCLOCK_CC_MULT                (1 << PTP_VCLOCK_CC_SHIFT)
#define PTP_VCLOCK_FADJ_SHIFT                9
#define PTP_VCLOCK_FADJ_DENOMINATOR        15625ULL
#define PTP_VCLOCK_REFRESH_INTERVAL        (HZ * 2)

/* protects vclock_hash addition/deletion */
static DEFINE_SPINLOCK(vclock_hash_lock);

static DEFINE_READ_MOSTLY_HASHTABLE(vclock_hash, 8);

static void ptp_vclock_hash_add(struct ptp_vclock *vclock)
{
        spin_lock(&vclock_hash_lock);

        hlist_add_head_rcu(&vclock->vclock_hash_node,
                           &vclock_hash[vclock->clock->index % HASH_SIZE(vclock_hash)]);

        spin_unlock(&vclock_hash_lock);
}

static void ptp_vclock_hash_del(struct ptp_vclock *vclock)
{
        spin_lock(&vclock_hash_lock);

        hlist_del_init_rcu(&vclock->vclock_hash_node);

        spin_unlock(&vclock_hash_lock);

        synchronize_rcu();
}

static int ptp_vclock_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
{
        struct ptp_vclock *vclock = info_to_vclock(ptp);
        s64 adj;

        adj = (s64)scaled_ppm << PTP_VCLOCK_FADJ_SHIFT;
        adj = div_s64(adj, PTP_VCLOCK_FADJ_DENOMINATOR);

        if (mutex_lock_interruptible(&vclock->lock))
                return -EINTR;
        timecounter_read(&vclock->tc);
        vclock->cc.mult = PTP_VCLOCK_CC_MULT + adj;
        mutex_unlock(&vclock->lock);

        return 0;
}

static int ptp_vclock_adjtime(struct ptp_clock_info *ptp, s64 delta)
{
        struct ptp_vclock *vclock = info_to_vclock(ptp);

        if (mutex_lock_interruptible(&vclock->lock))
                return -EINTR;
        timecounter_adjtime(&vclock->tc, delta);
        mutex_unlock(&vclock->lock);

        return 0;
}

static int ptp_vclock_gettime(struct ptp_clock_info *ptp,
                              struct timespec64 *ts)
{
        struct ptp_vclock *vclock = info_to_vclock(ptp);
        u64 ns;

        if (mutex_lock_interruptible(&vclock->lock))
                return -EINTR;
        ns = timecounter_read(&vclock->tc);
        mutex_unlock(&vclock->lock);
        *ts = ns_to_timespec64(ns);

        return 0;
}

static int ptp_vclock_gettimex(struct ptp_clock_info *ptp,
                               struct timespec64 *ts,
                               struct ptp_system_timestamp *sts)
{
        struct ptp_vclock *vclock = info_to_vclock(ptp);
        struct ptp_clock *pptp = vclock->pclock;
        struct timespec64 pts;
        int err;
        u64 ns;

        err = pptp->info->getcyclesx64(pptp->info, &pts, sts);
        if (err)
                return err;

        if (mutex_lock_interruptible(&vclock->lock))
                return -EINTR;
        ns = timecounter_cyc2time(&vclock->tc, timespec64_to_ns(&pts));
        mutex_unlock(&vclock->lock);

        *ts = ns_to_timespec64(ns);

        return 0;
}

static int ptp_vclock_settime(struct ptp_clock_info *ptp,
                              const struct timespec64 *ts)
{
        struct ptp_vclock *vclock = info_to_vclock(ptp);
        u64 ns = timespec64_to_ns(ts);

        if (mutex_lock_interruptible(&vclock->lock))
                return -EINTR;
        timecounter_init(&vclock->tc, &vclock->cc, ns);
        mutex_unlock(&vclock->lock);

        return 0;
}

static int ptp_vclock_getcrosststamp(struct ptp_clock_info *ptp,
                                     struct system_device_crosststamp *xtstamp)
{
        struct ptp_vclock *vclock = info_to_vclock(ptp);
        struct ptp_clock *pptp = vclock->pclock;
        int err;
        u64 ns;

        err = pptp->info->getcrosscycles(pptp->info, xtstamp);
        if (err)
                return err;

        if (mutex_lock_interruptible(&vclock->lock))
                return -EINTR;
        ns = timecounter_cyc2time(&vclock->tc, ktime_to_ns(xtstamp->device));
        mutex_unlock(&vclock->lock);

        xtstamp->device = ns_to_ktime(ns);

        return 0;
}

static long ptp_vclock_refresh(struct ptp_clock_info *ptp)
{
        struct ptp_vclock *vclock = info_to_vclock(ptp);
        struct timespec64 ts;

        ptp_vclock_gettime(&vclock->info, &ts);

        return PTP_VCLOCK_REFRESH_INTERVAL;
}

static const struct ptp_clock_info ptp_vclock_info = {
        .owner                = THIS_MODULE,
        .name                = "ptp virtual clock",
        .max_adj        = 500000000,
        .adjfine        = ptp_vclock_adjfine,
        .adjtime        = ptp_vclock_adjtime,
        .settime64        = ptp_vclock_settime,
        .do_aux_work        = ptp_vclock_refresh,
};

static u64 ptp_vclock_read(const struct cyclecounter *cc)
{
        struct ptp_vclock *vclock = cc_to_vclock(cc);
        struct ptp_clock *ptp = vclock->pclock;
        struct timespec64 ts = {};

        ptp->info->getcycles64(ptp->info, &ts);

        return timespec64_to_ns(&ts);
}

static const struct cyclecounter ptp_vclock_cc = {
        .read        = ptp_vclock_read,
        .mask        = CYCLECOUNTER_MASK(32),
        .mult        = PTP_VCLOCK_CC_MULT,
        .shift        = PTP_VCLOCK_CC_SHIFT,
};

struct ptp_vclock *ptp_vclock_register(struct ptp_clock *pclock)
{
        struct ptp_vclock *vclock;

        vclock = kzalloc(sizeof(*vclock), GFP_KERNEL);
        if (!vclock)
                return NULL;

        vclock->pclock = pclock;
        vclock->info = ptp_vclock_info;
        if (pclock->info->getcyclesx64)
                vclock->info.gettimex64 = ptp_vclock_gettimex;
        else
                vclock->info.gettime64 = ptp_vclock_gettime;
        if (pclock->info->getcrosscycles)
                vclock->info.getcrosststamp = ptp_vclock_getcrosststamp;
        vclock->cc = ptp_vclock_cc;

        snprintf(vclock->info.name, PTP_CLOCK_NAME_LEN, "ptp%d_virt",
                 pclock->index);

        INIT_HLIST_NODE(&vclock->vclock_hash_node);

        mutex_init(&vclock->lock);

        vclock->clock = ptp_clock_register(&vclock->info, &pclock->dev);
        if (IS_ERR_OR_NULL(vclock->clock)) {
                kfree(vclock);
                return NULL;
        }

        timecounter_init(&vclock->tc, &vclock->cc, 0);
        ptp_schedule_worker(vclock->clock, PTP_VCLOCK_REFRESH_INTERVAL);

        ptp_vclock_hash_add(vclock);

        return vclock;
}

void ptp_vclock_unregister(struct ptp_vclock *vclock)
{
        ptp_vclock_hash_del(vclock);

        ptp_clock_unregister(vclock->clock);
        kfree(vclock);
}

#if IS_BUILTIN(CONFIG_PTP_1588_CLOCK)
int ptp_get_vclocks_index(int pclock_index, int **vclock_index)
{
        char name[PTP_CLOCK_NAME_LEN] = "";
        struct ptp_clock *ptp;
        struct device *dev;
        int num = 0;

        if (pclock_index < 0)
                return num;

        snprintf(name, PTP_CLOCK_NAME_LEN, "ptp%d", pclock_index);
        dev = class_find_device_by_name(&ptp_class, name);
        if (!dev)
                return num;

        ptp = dev_get_drvdata(dev);

        if (mutex_lock_interruptible(&ptp->n_vclocks_mux)) {
                put_device(dev);
                return num;
        }

        *vclock_index = kzalloc(sizeof(int) * ptp->n_vclocks, GFP_KERNEL);
        if (!(*vclock_index))
                goto out;

        memcpy(*vclock_index, ptp->vclock_index, sizeof(int) * ptp->n_vclocks);
        num = ptp->n_vclocks;
out:
        mutex_unlock(&ptp->n_vclocks_mux);
        put_device(dev);
        return num;
}
EXPORT_SYMBOL(ptp_get_vclocks_index);

ktime_t ptp_convert_timestamp(const ktime_t *hwtstamp, int vclock_index)
{
        unsigned int hash = vclock_index % HASH_SIZE(vclock_hash);
        struct ptp_vclock *vclock;
        u64 ns;
        u64 vclock_ns = 0;

        ns = ktime_to_ns(*hwtstamp);

        rcu_read_lock();

        hlist_for_each_entry_rcu(vclock, &vclock_hash[hash], vclock_hash_node) {
                if (vclock->clock->index != vclock_index)
                        continue;

                if (mutex_lock_interruptible(&vclock->lock))
                        break;
                vclock_ns = timecounter_cyc2time(&vclock->tc, ns);
                mutex_unlock(&vclock->lock);
                break;
        }

        rcu_read_unlock();

        return ns_to_ktime(vclock_ns);
}
EXPORT_SYMBOL(ptp_convert_timestamp);
#endif

































    6 








    6 

    1 



    6 
    6 


    6 



    6 
    6 








    5 














    5 









    5 


    5 
    4 











    6 





    6 






    6 








    6 
    6 


    6 



    6 




    6 




    6 
    6 
































    2 

    2 


    2 

    2 





    2 


    2 






















    3 
    1 




   10 
    2 
    1 
















    7 

    1 
    1 





    6 




    1 



    6 



    1 
    1 
    1 



    5 







    6 





    5 

    5 
    5 



    5 

    2 









   11 


   11 
    3 


    9 

    8 





    7 











    7 


    5 


    4 





    7 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
// SPDX-License-Identifier: GPL-2.0-only

#include "netlink.h"
#include "common.h"
#include "bitset.h"

/* LINKMODES_GET */

struct linkmodes_req_info {
        struct ethnl_req_info                base;
};

struct linkmodes_reply_data {
        struct ethnl_reply_data                base;
        struct ethtool_link_ksettings        ksettings;
        struct ethtool_link_settings        *lsettings;
        bool                                peer_empty;
};

#define LINKMODES_REPDATA(__reply_base) \
        container_of(__reply_base, struct linkmodes_reply_data, base)

const struct nla_policy ethnl_linkmodes_get_policy[] = {
        [ETHTOOL_A_LINKMODES_HEADER]                =
                NLA_POLICY_NESTED(ethnl_header_policy),
};

static int linkmodes_prepare_data(const struct ethnl_req_info *req_base,
                                  struct ethnl_reply_data *reply_base,
                                  const struct genl_info *info)
{
        struct linkmodes_reply_data *data = LINKMODES_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        int ret;

        data->lsettings = &data->ksettings.base;

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                return ret;

        ret = __ethtool_get_link_ksettings(dev, &data->ksettings);
        if (ret < 0) {
                GENL_SET_ERR_MSG(info, "failed to retrieve link settings");
                goto out;
        }

        if (!dev->ethtool_ops->cap_link_lanes_supported)
                data->ksettings.lanes = 0;

        data->peer_empty =
                bitmap_empty(data->ksettings.link_modes.lp_advertising,
                             __ETHTOOL_LINK_MODE_MASK_NBITS);

out:
        ethnl_ops_complete(dev);
        return ret;
}

static int linkmodes_reply_size(const struct ethnl_req_info *req_base,
                                const struct ethnl_reply_data *reply_base)
{
        const struct linkmodes_reply_data *data = LINKMODES_REPDATA(reply_base);
        const struct ethtool_link_ksettings *ksettings = &data->ksettings;
        const struct ethtool_link_settings *lsettings = &ksettings->base;
        bool compact = req_base->flags & ETHTOOL_FLAG_COMPACT_BITSETS;
        int len, ret;

        len = nla_total_size(sizeof(u8)) /* LINKMODES_AUTONEG */
                + nla_total_size(sizeof(u32)) /* LINKMODES_SPEED */
                + nla_total_size(sizeof(u32)) /* LINKMODES_LANES */
                + nla_total_size(sizeof(u8)) /* LINKMODES_DUPLEX */
                + nla_total_size(sizeof(u8)) /* LINKMODES_RATE_MATCHING */
                + 0;
        ret = ethnl_bitset_size(ksettings->link_modes.advertising,
                                ksettings->link_modes.supported,
                                __ETHTOOL_LINK_MODE_MASK_NBITS,
                                link_mode_names, compact);
        if (ret < 0)
                return ret;
        len += ret;
        if (!data->peer_empty) {
                ret = ethnl_bitset_size(ksettings->link_modes.lp_advertising,
                                        NULL, __ETHTOOL_LINK_MODE_MASK_NBITS,
                                        link_mode_names, compact);
                if (ret < 0)
                        return ret;
                len += ret;
        }

        if (lsettings->master_slave_cfg != MASTER_SLAVE_CFG_UNSUPPORTED)
                len += nla_total_size(sizeof(u8));

        if (lsettings->master_slave_state != MASTER_SLAVE_STATE_UNSUPPORTED)
                len += nla_total_size(sizeof(u8));

        return len;
}

static int linkmodes_fill_reply(struct sk_buff *skb,
                                const struct ethnl_req_info *req_base,
                                const struct ethnl_reply_data *reply_base)
{
        const struct linkmodes_reply_data *data = LINKMODES_REPDATA(reply_base);
        const struct ethtool_link_ksettings *ksettings = &data->ksettings;
        const struct ethtool_link_settings *lsettings = &ksettings->base;
        bool compact = req_base->flags & ETHTOOL_FLAG_COMPACT_BITSETS;
        int ret;

        if (nla_put_u8(skb, ETHTOOL_A_LINKMODES_AUTONEG, lsettings->autoneg))
                return -EMSGSIZE;

        ret = ethnl_put_bitset(skb, ETHTOOL_A_LINKMODES_OURS,
                               ksettings->link_modes.advertising,
                               ksettings->link_modes.supported,
                               __ETHTOOL_LINK_MODE_MASK_NBITS, link_mode_names,
                               compact);
        if (ret < 0)
                return -EMSGSIZE;
        if (!data->peer_empty) {
                ret = ethnl_put_bitset(skb, ETHTOOL_A_LINKMODES_PEER,
                                       ksettings->link_modes.lp_advertising,
                                       NULL, __ETHTOOL_LINK_MODE_MASK_NBITS,
                                       link_mode_names, compact);
                if (ret < 0)
                        return -EMSGSIZE;
        }

        if (nla_put_u32(skb, ETHTOOL_A_LINKMODES_SPEED, lsettings->speed) ||
            nla_put_u8(skb, ETHTOOL_A_LINKMODES_DUPLEX, lsettings->duplex))
                return -EMSGSIZE;

        if (ksettings->lanes &&
            nla_put_u32(skb, ETHTOOL_A_LINKMODES_LANES, ksettings->lanes))
                return -EMSGSIZE;

        if (lsettings->master_slave_cfg != MASTER_SLAVE_CFG_UNSUPPORTED &&
            nla_put_u8(skb, ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG,
                       lsettings->master_slave_cfg))
                return -EMSGSIZE;

        if (lsettings->master_slave_state != MASTER_SLAVE_STATE_UNSUPPORTED &&
            nla_put_u8(skb, ETHTOOL_A_LINKMODES_MASTER_SLAVE_STATE,
                       lsettings->master_slave_state))
                return -EMSGSIZE;

        if (nla_put_u8(skb, ETHTOOL_A_LINKMODES_RATE_MATCHING,
                       lsettings->rate_matching))
                return -EMSGSIZE;

        return 0;
}

/* LINKMODES_SET */

const struct nla_policy ethnl_linkmodes_set_policy[] = {
        [ETHTOOL_A_LINKMODES_HEADER]                =
                NLA_POLICY_NESTED(ethnl_header_policy),
        [ETHTOOL_A_LINKMODES_AUTONEG]                = { .type = NLA_U8 },
        [ETHTOOL_A_LINKMODES_OURS]                = { .type = NLA_NESTED },
        [ETHTOOL_A_LINKMODES_SPEED]                = { .type = NLA_U32 },
        [ETHTOOL_A_LINKMODES_DUPLEX]                = { .type = NLA_U8 },
        [ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG]        = { .type = NLA_U8 },
        [ETHTOOL_A_LINKMODES_LANES]                = NLA_POLICY_RANGE(NLA_U32, 1, 8),
};

/* Set advertised link modes to all supported modes matching requested speed,
 * lanes and duplex values. Called when autonegotiation is on, speed, lanes or
 * duplex is requested but no link mode change. This is done in userspace with
 * ioctl() interface, move it into kernel for netlink.
 * Returns true if advertised modes bitmap was modified.
 */
static bool ethnl_auto_linkmodes(struct ethtool_link_ksettings *ksettings,
                                 bool req_speed, bool req_lanes, bool req_duplex)
{
        unsigned long *advertising = ksettings->link_modes.advertising;
        unsigned long *supported = ksettings->link_modes.supported;
        DECLARE_BITMAP(old_adv, __ETHTOOL_LINK_MODE_MASK_NBITS);
        unsigned int i;

        bitmap_copy(old_adv, advertising, __ETHTOOL_LINK_MODE_MASK_NBITS);

        for (i = 0; i < __ETHTOOL_LINK_MODE_MASK_NBITS; i++) {
                const struct link_mode_info *info = &link_mode_params[i];

                if (info->speed == SPEED_UNKNOWN)
                        continue;
                if (test_bit(i, supported) &&
                    (!req_speed || info->speed == ksettings->base.speed) &&
                    (!req_lanes || info->lanes == ksettings->lanes) &&
                    (!req_duplex || info->duplex == ksettings->base.duplex))
                        set_bit(i, advertising);
                else
                        clear_bit(i, advertising);
        }

        return !bitmap_equal(old_adv, advertising,
                             __ETHTOOL_LINK_MODE_MASK_NBITS);
}

static bool ethnl_validate_master_slave_cfg(u8 cfg)
{
        switch (cfg) {
        case MASTER_SLAVE_CFG_MASTER_PREFERRED:
        case MASTER_SLAVE_CFG_SLAVE_PREFERRED:
        case MASTER_SLAVE_CFG_MASTER_FORCE:
        case MASTER_SLAVE_CFG_SLAVE_FORCE:
                return true;
        }

        return false;
}

static int ethnl_check_linkmodes(struct genl_info *info, struct nlattr **tb)
{
        const struct nlattr *master_slave_cfg, *lanes_cfg;

        master_slave_cfg = tb[ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG];
        if (master_slave_cfg &&
            !ethnl_validate_master_slave_cfg(nla_get_u8(master_slave_cfg))) {
                NL_SET_ERR_MSG_ATTR(info->extack, master_slave_cfg,
                                    "master/slave value is invalid");
                return -EOPNOTSUPP;
        }

        lanes_cfg = tb[ETHTOOL_A_LINKMODES_LANES];
        if (lanes_cfg && !is_power_of_2(nla_get_u32(lanes_cfg))) {
                NL_SET_ERR_MSG_ATTR(info->extack, lanes_cfg,
                                    "lanes value is invalid");
                return -EINVAL;
        }

        return 0;
}

static int ethnl_update_linkmodes(struct genl_info *info, struct nlattr **tb,
                                  struct ethtool_link_ksettings *ksettings,
                                  bool *mod, const struct net_device *dev)
{
        struct ethtool_link_settings *lsettings = &ksettings->base;
        bool req_speed, req_lanes, req_duplex;
        const struct nlattr *master_slave_cfg, *lanes_cfg;
        int ret;

        master_slave_cfg = tb[ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG];
        if (master_slave_cfg) {
                if (lsettings->master_slave_cfg == MASTER_SLAVE_CFG_UNSUPPORTED) {
                        NL_SET_ERR_MSG_ATTR(info->extack, master_slave_cfg,
                                            "master/slave configuration not supported by device");
                        return -EOPNOTSUPP;
                }
        }

        *mod = false;
        req_speed = tb[ETHTOOL_A_LINKMODES_SPEED];
        req_lanes = tb[ETHTOOL_A_LINKMODES_LANES];
        req_duplex = tb[ETHTOOL_A_LINKMODES_DUPLEX];

        ethnl_update_u8(&lsettings->autoneg, tb[ETHTOOL_A_LINKMODES_AUTONEG],
                        mod);

        lanes_cfg = tb[ETHTOOL_A_LINKMODES_LANES];
        if (lanes_cfg) {
                /* If autoneg is off and lanes parameter is not supported by the
                 * driver, return an error.
                 */
                if (!lsettings->autoneg &&
                    !dev->ethtool_ops->cap_link_lanes_supported) {
                        NL_SET_ERR_MSG_ATTR(info->extack, lanes_cfg,
                                            "lanes configuration not supported by device");
                        return -EOPNOTSUPP;
                }
        } else if (!lsettings->autoneg && ksettings->lanes) {
                /* If autoneg is off and lanes parameter is not passed from user but
                 * it was defined previously then set the lanes parameter to 0.
                 */
                ksettings->lanes = 0;
                *mod = true;
        }

        ret = ethnl_update_bitset(ksettings->link_modes.advertising,
                                  __ETHTOOL_LINK_MODE_MASK_NBITS,
                                  tb[ETHTOOL_A_LINKMODES_OURS], link_mode_names,
                                  info->extack, mod);
        if (ret < 0)
                return ret;
        ethnl_update_u32(&lsettings->speed, tb[ETHTOOL_A_LINKMODES_SPEED],
                         mod);
        ethnl_update_u32(&ksettings->lanes, lanes_cfg, mod);
        ethnl_update_u8(&lsettings->duplex, tb[ETHTOOL_A_LINKMODES_DUPLEX],
                        mod);
        ethnl_update_u8(&lsettings->master_slave_cfg, master_slave_cfg, mod);

        if (!tb[ETHTOOL_A_LINKMODES_OURS] && lsettings->autoneg &&
            (req_speed || req_lanes || req_duplex) &&
            ethnl_auto_linkmodes(ksettings, req_speed, req_lanes, req_duplex))
                *mod = true;

        return 0;
}

static int
ethnl_set_linkmodes_validate(struct ethnl_req_info *req_info,
                             struct genl_info *info)
{
        const struct ethtool_ops *ops = req_info->dev->ethtool_ops;
        int ret;

        ret = ethnl_check_linkmodes(info, info->attrs);
        if (ret < 0)
                return ret;

        if (!ops->get_link_ksettings || !ops->set_link_ksettings)
                return -EOPNOTSUPP;
        return 1;
}

static int
ethnl_set_linkmodes(struct ethnl_req_info *req_info, struct genl_info *info)
{
        struct ethtool_link_ksettings ksettings = {};
        struct net_device *dev = req_info->dev;
        struct nlattr **tb = info->attrs;
        bool mod = false;
        int ret;

        ret = __ethtool_get_link_ksettings(dev, &ksettings);
        if (ret < 0) {
                GENL_SET_ERR_MSG(info, "failed to retrieve link settings");
                return ret;
        }

        ret = ethnl_update_linkmodes(info, tb, &ksettings, &mod, dev);
        if (ret < 0)
                return ret;
        if (!mod)
                return 0;

        ret = dev->ethtool_ops->set_link_ksettings(dev, &ksettings);
        if (ret < 0) {
                GENL_SET_ERR_MSG(info, "link settings update failed");
                return ret;
        }

        return 1;
}

const struct ethnl_request_ops ethnl_linkmodes_request_ops = {
        .request_cmd                = ETHTOOL_MSG_LINKMODES_GET,
        .reply_cmd                = ETHTOOL_MSG_LINKMODES_GET_REPLY,
        .hdr_attr                = ETHTOOL_A_LINKMODES_HEADER,
        .req_info_size                = sizeof(struct linkmodes_req_info),
        .reply_data_size        = sizeof(struct linkmodes_reply_data),

        .prepare_data                = linkmodes_prepare_data,
        .reply_size                = linkmodes_reply_size,
        .fill_reply                = linkmodes_fill_reply,

        .set_validate                = ethnl_set_linkmodes_validate,
        .set                        = ethnl_set_linkmodes,
        .set_ntf_cmd                = ETHTOOL_MSG_LINKMODES_NTF,
};
































  148 



  148 





























































































   68 


   68 
   68 

   68 









   68 



   68 







   68 












   68 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
#include <linux/rtnetlink.h>
#include <linux/notifier.h>
#include <linux/rcupdate.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/fib_notifier.h>

static unsigned int fib_notifier_net_id;

struct fib_notifier_net {
        struct list_head fib_notifier_ops;
        struct atomic_notifier_head fib_chain;
};

int call_fib_notifier(struct notifier_block *nb,
                      enum fib_event_type event_type,
                      struct fib_notifier_info *info)
{
        int err;

        err = nb->notifier_call(nb, event_type, info);
        return notifier_to_errno(err);
}
EXPORT_SYMBOL(call_fib_notifier);

int call_fib_notifiers(struct net *net, enum fib_event_type event_type,
                       struct fib_notifier_info *info)
{
        struct fib_notifier_net *fn_net = net_generic(net, fib_notifier_net_id);
        int err;

        err = atomic_notifier_call_chain(&fn_net->fib_chain, event_type, info);
        return notifier_to_errno(err);
}
EXPORT_SYMBOL(call_fib_notifiers);

static unsigned int fib_seq_sum(struct net *net)
{
        struct fib_notifier_net *fn_net = net_generic(net, fib_notifier_net_id);
        struct fib_notifier_ops *ops;
        unsigned int fib_seq = 0;

        rcu_read_lock();
        list_for_each_entry_rcu(ops, &fn_net->fib_notifier_ops, list) {
                if (!try_module_get(ops->owner))
                        continue;
                fib_seq += ops->fib_seq_read(net);
                module_put(ops->owner);
        }
        rcu_read_unlock();

        return fib_seq;
}

static int fib_net_dump(struct net *net, struct notifier_block *nb,
                        struct netlink_ext_ack *extack)
{
        struct fib_notifier_net *fn_net = net_generic(net, fib_notifier_net_id);
        struct fib_notifier_ops *ops;
        int err = 0;

        rcu_read_lock();
        list_for_each_entry_rcu(ops, &fn_net->fib_notifier_ops, list) {
                if (!try_module_get(ops->owner))
                        continue;
                err = ops->fib_dump(net, nb, extack);
                module_put(ops->owner);
                if (err)
                        goto unlock;
        }

unlock:
        rcu_read_unlock();

        return err;
}

static bool fib_dump_is_consistent(struct net *net, struct notifier_block *nb,
                                   void (*cb)(struct notifier_block *nb),
                                   unsigned int fib_seq)
{
        struct fib_notifier_net *fn_net = net_generic(net, fib_notifier_net_id);

        atomic_notifier_chain_register(&fn_net->fib_chain, nb);
        if (fib_seq == fib_seq_sum(net))
                return true;
        atomic_notifier_chain_unregister(&fn_net->fib_chain, nb);
        if (cb)
                cb(nb);
        return false;
}

#define FIB_DUMP_MAX_RETRIES 5
int register_fib_notifier(struct net *net, struct notifier_block *nb,
                          void (*cb)(struct notifier_block *nb),
                          struct netlink_ext_ack *extack)
{
        int retries = 0;
        int err;

        do {
                unsigned int fib_seq = fib_seq_sum(net);

                err = fib_net_dump(net, nb, extack);
                if (err)
                        return err;

                if (fib_dump_is_consistent(net, nb, cb, fib_seq))
                        return 0;
        } while (++retries < FIB_DUMP_MAX_RETRIES);

        return -EBUSY;
}
EXPORT_SYMBOL(register_fib_notifier);

int unregister_fib_notifier(struct net *net, struct notifier_block *nb)
{
        struct fib_notifier_net *fn_net = net_generic(net, fib_notifier_net_id);

        return atomic_notifier_chain_unregister(&fn_net->fib_chain, nb);
}
EXPORT_SYMBOL(unregister_fib_notifier);

static int __fib_notifier_ops_register(struct fib_notifier_ops *ops,
                                       struct net *net)
{
        struct fib_notifier_net *fn_net = net_generic(net, fib_notifier_net_id);
        struct fib_notifier_ops *o;

        list_for_each_entry(o, &fn_net->fib_notifier_ops, list)
                if (ops->family == o->family)
                        return -EEXIST;
        list_add_tail_rcu(&ops->list, &fn_net->fib_notifier_ops);
        return 0;
}

struct fib_notifier_ops *
fib_notifier_ops_register(const struct fib_notifier_ops *tmpl, struct net *net)
{
        struct fib_notifier_ops *ops;
        int err;

        ops = kmemdup(tmpl, sizeof(*ops), GFP_KERNEL);
        if (!ops)
                return ERR_PTR(-ENOMEM);

        err = __fib_notifier_ops_register(ops, net);
        if (err)
                goto err_register;

        return ops;

err_register:
        kfree(ops);
        return ERR_PTR(err);
}
EXPORT_SYMBOL(fib_notifier_ops_register);

void fib_notifier_ops_unregister(struct fib_notifier_ops *ops)
{
        list_del_rcu(&ops->list);
        kfree_rcu(ops, rcu);
}
EXPORT_SYMBOL(fib_notifier_ops_unregister);

static int __net_init fib_notifier_net_init(struct net *net)
{
        struct fib_notifier_net *fn_net = net_generic(net, fib_notifier_net_id);

        INIT_LIST_HEAD(&fn_net->fib_notifier_ops);
        ATOMIC_INIT_NOTIFIER_HEAD(&fn_net->fib_chain);
        return 0;
}

static void __net_exit fib_notifier_net_exit(struct net *net)
{
        struct fib_notifier_net *fn_net = net_generic(net, fib_notifier_net_id);

        WARN_ON_ONCE(!list_empty(&fn_net->fib_notifier_ops));
}

static struct pernet_operations fib_notifier_net_ops = {
        .init = fib_notifier_net_init,
        .exit = fib_notifier_net_exit,
        .id = &fib_notifier_net_id,
        .size = sizeof(struct fib_notifier_net),
};

static int __init fib_notifier_init(void)
{
        return register_pernet_subsys(&fib_notifier_net_ops);
}

subsys_initcall(fib_notifier_init);














































































































































































































































































  168 


















  343 


























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
/* SPDX-License-Identifier: GPL-2.0 OR MIT */
#ifndef __LINUX_OVERFLOW_H
#define __LINUX_OVERFLOW_H

#include <linux/compiler.h>
#include <linux/limits.h>
#include <linux/const.h>

/*
 * We need to compute the minimum and maximum values representable in a given
 * type. These macros may also be useful elsewhere. It would seem more obvious
 * to do something like:
 *
 * #define type_min(T) (T)(is_signed_type(T) ? (T)1 << (8*sizeof(T)-1) : 0)
 * #define type_max(T) (T)(is_signed_type(T) ? ((T)1 << (8*sizeof(T)-1)) - 1 : ~(T)0)
 *
 * Unfortunately, the middle expressions, strictly speaking, have
 * undefined behaviour, and at least some versions of gcc warn about
 * the type_max expression (but not if -fsanitize=undefined is in
 * effect; in that case, the warning is deferred to runtime...).
 *
 * The slightly excessive casting in type_min is to make sure the
 * macros also produce sensible values for the exotic type _Bool. [The
 * overflow checkers only almost work for _Bool, but that's
 * a-feature-not-a-bug, since people shouldn't be doing arithmetic on
 * _Bools. Besides, the gcc builtins don't allow _Bool* as third
 * argument.]
 *
 * Idea stolen from
 * https://mail-index.netbsd.org/tech-misc/2007/02/05/0000.html -
 * credit to Christian Biere.
 */
#define __type_half_max(type) ((type)1 << (8*sizeof(type) - 1 - is_signed_type(type)))
#define __type_max(T) ((T)((__type_half_max(T) - 1) + __type_half_max(T)))
#define type_max(t)        __type_max(typeof(t))
#define __type_min(T) ((T)((T)-type_max(T)-(T)1))
#define type_min(t)        __type_min(typeof(t))

/*
 * Avoids triggering -Wtype-limits compilation warning,
 * while using unsigned data types to check a < 0.
 */
#define is_non_negative(a) ((a) > 0 || (a) == 0)
#define is_negative(a) (!(is_non_negative(a)))

/*
 * Allows for effectively applying __must_check to a macro so we can have
 * both the type-agnostic benefits of the macros while also being able to
 * enforce that the return value is, in fact, checked.
 */
static inline bool __must_check __must_check_overflow(bool overflow)
{
        return unlikely(overflow);
}

/**
 * check_add_overflow() - Calculate addition with overflow checking
 * @a: first addend
 * @b: second addend
 * @d: pointer to store sum
 *
 * Returns true on wrap-around, false otherwise.
 *
 * *@d holds the results of the attempted addition, regardless of whether
 * wrap-around occurred.
 */
#define check_add_overflow(a, b, d)        \
        __must_check_overflow(__builtin_add_overflow(a, b, d))

/**
 * wrapping_add() - Intentionally perform a wrapping addition
 * @type: type for result of calculation
 * @a: first addend
 * @b: second addend
 *
 * Return the potentially wrapped-around addition without
 * tripping any wrap-around sanitizers that may be enabled.
 */
#define wrapping_add(type, a, b)                                \
        ({                                                        \
                type __val;                                        \
                __builtin_add_overflow(a, b, &__val);                \
                __val;                                                \
        })

/**
 * wrapping_assign_add() - Intentionally perform a wrapping increment assignment
 * @var: variable to be incremented
 * @offset: amount to add
 *
 * Increments @var by @offset with wrap-around. Returns the resulting
 * value of @var. Will not trip any wrap-around sanitizers.
 *
 * Returns the new value of @var.
 */
#define wrapping_assign_add(var, offset)                                \
        ({                                                                \
                typeof(var) *__ptr = &(var);                                \
                *__ptr = wrapping_add(typeof(var), *__ptr, offset);        \
        })

/**
 * check_sub_overflow() - Calculate subtraction with overflow checking
 * @a: minuend; value to subtract from
 * @b: subtrahend; value to subtract from @a
 * @d: pointer to store difference
 *
 * Returns true on wrap-around, false otherwise.
 *
 * *@d holds the results of the attempted subtraction, regardless of whether
 * wrap-around occurred.
 */
#define check_sub_overflow(a, b, d)        \
        __must_check_overflow(__builtin_sub_overflow(a, b, d))

/**
 * wrapping_sub() - Intentionally perform a wrapping subtraction
 * @type: type for result of calculation
 * @a: minuend; value to subtract from
 * @b: subtrahend; value to subtract from @a
 *
 * Return the potentially wrapped-around subtraction without
 * tripping any wrap-around sanitizers that may be enabled.
 */
#define wrapping_sub(type, a, b)                                \
        ({                                                        \
                type __val;                                        \
                __builtin_sub_overflow(a, b, &__val);                \
                __val;                                                \
        })

/**
 * wrapping_assign_sub() - Intentionally perform a wrapping decrement assign
 * @var: variable to be decremented
 * @offset: amount to subtract
 *
 * Decrements @var by @offset with wrap-around. Returns the resulting
 * value of @var. Will not trip any wrap-around sanitizers.
 *
 * Returns the new value of @var.
 */
#define wrapping_assign_sub(var, offset)                                \
        ({                                                                \
                typeof(var) *__ptr = &(var);                                \
                *__ptr = wrapping_sub(typeof(var), *__ptr, offset);        \
        })

/**
 * check_mul_overflow() - Calculate multiplication with overflow checking
 * @a: first factor
 * @b: second factor
 * @d: pointer to store product
 *
 * Returns true on wrap-around, false otherwise.
 *
 * *@d holds the results of the attempted multiplication, regardless of whether
 * wrap-around occurred.
 */
#define check_mul_overflow(a, b, d)        \
        __must_check_overflow(__builtin_mul_overflow(a, b, d))

/**
 * wrapping_mul() - Intentionally perform a wrapping multiplication
 * @type: type for result of calculation
 * @a: first factor
 * @b: second factor
 *
 * Return the potentially wrapped-around multiplication without
 * tripping any wrap-around sanitizers that may be enabled.
 */
#define wrapping_mul(type, a, b)                                \
        ({                                                        \
                type __val;                                        \
                __builtin_mul_overflow(a, b, &__val);                \
                __val;                                                \
        })

/**
 * check_shl_overflow() - Calculate a left-shifted value and check overflow
 * @a: Value to be shifted
 * @s: How many bits left to shift
 * @d: Pointer to where to store the result
 *
 * Computes *@d = (@a << @s)
 *
 * Returns true if '*@d' cannot hold the result or when '@a << @s' doesn't
 * make sense. Example conditions:
 *
 * - '@a << @s' causes bits to be lost when stored in *@d.
 * - '@s' is garbage (e.g. negative) or so large that the result of
 *   '@a << @s' is guaranteed to be 0.
 * - '@a' is negative.
 * - '@a << @s' sets the sign bit, if any, in '*@d'.
 *
 * '*@d' will hold the results of the attempted shift, but is not
 * considered "safe for use" if true is returned.
 */
#define check_shl_overflow(a, s, d) __must_check_overflow(({                \
        typeof(a) _a = a;                                                \
        typeof(s) _s = s;                                                \
        typeof(d) _d = d;                                                \
        unsigned long long _a_full = _a;                                \
        unsigned int _to_shift =                                        \
                is_non_negative(_s) && _s < 8 * sizeof(*d) ? _s : 0;        \
        *_d = (_a_full << _to_shift);                                        \
        (_to_shift != _s || is_negative(*_d) || is_negative(_a) ||        \
        (*_d >> _to_shift) != _a);                                        \
}))

#define __overflows_type_constexpr(x, T) (                        \
        is_unsigned_type(typeof(x)) ?                                \
                (x) > type_max(T) :                                \
        is_unsigned_type(typeof(T)) ?                                \
                (x) < 0 || (x) > type_max(T) :                        \
        (x) < type_min(T) || (x) > type_max(T))

#define __overflows_type(x, T)                ({        \
        typeof(T) v = 0;                        \
        check_add_overflow((x), v, &v);                \
})

/**
 * overflows_type - helper for checking the overflows between value, variables,
 *                    or data type
 *
 * @n: source constant value or variable to be checked
 * @T: destination variable or data type proposed to store @x
 *
 * Compares the @x expression for whether or not it can safely fit in
 * the storage of the type in @T. @x and @T can have different types.
 * If @x is a constant expression, this will also resolve to a constant
 * expression.
 *
 * Returns: true if overflow can occur, false otherwise.
 */
#define overflows_type(n, T)                                        \
        __builtin_choose_expr(__is_constexpr(n),                \
                              __overflows_type_constexpr(n, T),        \
                              __overflows_type(n, T))

/**
 * castable_to_type - like __same_type(), but also allows for casted literals
 *
 * @n: variable or constant value
 * @T: variable or data type
 *
 * Unlike the __same_type() macro, this allows a constant value as the
 * first argument. If this value would not overflow into an assignment
 * of the second argument's type, it returns true. Otherwise, this falls
 * back to __same_type().
 */
#define castable_to_type(n, T)                                                \
        __builtin_choose_expr(__is_constexpr(n),                        \
                              !__overflows_type_constexpr(n, T),        \
                              __same_type(n, T))

/**
 * size_mul() - Calculate size_t multiplication with saturation at SIZE_MAX
 * @factor1: first factor
 * @factor2: second factor
 *
 * Returns: calculate @factor1 * @factor2, both promoted to size_t,
 * with any overflow causing the return value to be SIZE_MAX. The
 * lvalue must be size_t to avoid implicit type conversion.
 */
static inline size_t __must_check size_mul(size_t factor1, size_t factor2)
{
        size_t bytes;

        if (check_mul_overflow(factor1, factor2, &bytes))
                return SIZE_MAX;

        return bytes;
}

/**
 * size_add() - Calculate size_t addition with saturation at SIZE_MAX
 * @addend1: first addend
 * @addend2: second addend
 *
 * Returns: calculate @addend1 + @addend2, both promoted to size_t,
 * with any overflow causing the return value to be SIZE_MAX. The
 * lvalue must be size_t to avoid implicit type conversion.
 */
static inline size_t __must_check size_add(size_t addend1, size_t addend2)
{
        size_t bytes;

        if (check_add_overflow(addend1, addend2, &bytes))
                return SIZE_MAX;

        return bytes;
}

/**
 * size_sub() - Calculate size_t subtraction with saturation at SIZE_MAX
 * @minuend: value to subtract from
 * @subtrahend: value to subtract from @minuend
 *
 * Returns: calculate @minuend - @subtrahend, both promoted to size_t,
 * with any overflow causing the return value to be SIZE_MAX. For
 * composition with the size_add() and size_mul() helpers, neither
 * argument may be SIZE_MAX (or the result with be forced to SIZE_MAX).
 * The lvalue must be size_t to avoid implicit type conversion.
 */
static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend)
{
        size_t bytes;

        if (minuend == SIZE_MAX || subtrahend == SIZE_MAX ||
            check_sub_overflow(minuend, subtrahend, &bytes))
                return SIZE_MAX;

        return bytes;
}

/**
 * array_size() - Calculate size of 2-dimensional array.
 * @a: dimension one
 * @b: dimension two
 *
 * Calculates size of 2-dimensional array: @a * @b.
 *
 * Returns: number of bytes needed to represent the array or SIZE_MAX on
 * overflow.
 */
#define array_size(a, b)        size_mul(a, b)

/**
 * array3_size() - Calculate size of 3-dimensional array.
 * @a: dimension one
 * @b: dimension two
 * @c: dimension three
 *
 * Calculates size of 3-dimensional array: @a * @b * @c.
 *
 * Returns: number of bytes needed to represent the array or SIZE_MAX on
 * overflow.
 */
#define array3_size(a, b, c)        size_mul(size_mul(a, b), c)

/**
 * flex_array_size() - Calculate size of a flexible array member
 *                     within an enclosing structure.
 * @p: Pointer to the structure.
 * @member: Name of the flexible array member.
 * @count: Number of elements in the array.
 *
 * Calculates size of a flexible array of @count number of @member
 * elements, at the end of structure @p.
 *
 * Return: number of bytes needed or SIZE_MAX on overflow.
 */
#define flex_array_size(p, member, count)                                \
        __builtin_choose_expr(__is_constexpr(count),                        \
                (count) * sizeof(*(p)->member) + __must_be_array((p)->member),        \
                size_mul(count, sizeof(*(p)->member) + __must_be_array((p)->member)))

/**
 * struct_size() - Calculate size of structure with trailing flexible array.
 * @p: Pointer to the structure.
 * @member: Name of the array member.
 * @count: Number of elements in the array.
 *
 * Calculates size of memory needed for structure of @p followed by an
 * array of @count number of @member elements.
 *
 * Return: number of bytes needed or SIZE_MAX on overflow.
 */
#define struct_size(p, member, count)                                        \
        __builtin_choose_expr(__is_constexpr(count),                        \
                sizeof(*(p)) + flex_array_size(p, member, count),        \
                size_add(sizeof(*(p)), flex_array_size(p, member, count)))

/**
 * struct_size_t() - Calculate size of structure with trailing flexible array
 * @type: structure type name.
 * @member: Name of the array member.
 * @count: Number of elements in the array.
 *
 * Calculates size of memory needed for structure @type followed by an
 * array of @count number of @member elements. Prefer using struct_size()
 * when possible instead, to keep calculations associated with a specific
 * instance variable of type @type.
 *
 * Return: number of bytes needed or SIZE_MAX on overflow.
 */
#define struct_size_t(type, member, count)                                        \
        struct_size((type *)NULL, member, count)

/**
 * _DEFINE_FLEX() - helper macro for DEFINE_FLEX() family.
 * Enables caller macro to pass (different) initializer.
 *
 * @type: structure type name, including "struct" keyword.
 * @name: Name for a variable to define.
 * @member: Name of the array member.
 * @count: Number of elements in the array; must be compile-time const.
 * @initializer: initializer expression (could be empty for no init).
 */
#define _DEFINE_FLEX(type, name, member, count, initializer...)                        \
        _Static_assert(__builtin_constant_p(count),                                \
                       "onstack flex array members require compile-time const count"); \
        union {                                                                        \
                u8 bytes[struct_size_t(type, member, count)];                        \
                type obj;                                                        \
        } name##_u initializer;                                                        \
        type *name = (type *)&name##_u

/**
 * DEFINE_RAW_FLEX() - Define an on-stack instance of structure with a trailing
 * flexible array member, when it does not have a __counted_by annotation.
 *
 * @type: structure type name, including "struct" keyword.
 * @name: Name for a variable to define.
 * @member: Name of the array member.
 * @count: Number of elements in the array; must be compile-time const.
 *
 * Define a zeroed, on-stack, instance of @type structure with a trailing
 * flexible array member.
 * Use __struct_size(@name) to get compile-time size of it afterwards.
 */
#define DEFINE_RAW_FLEX(type, name, member, count)        \
        _DEFINE_FLEX(type, name, member, count, = {})

/**
 * DEFINE_FLEX() - Define an on-stack instance of structure with a trailing
 * flexible array member.
 *
 * @TYPE: structure type name, including "struct" keyword.
 * @NAME: Name for a variable to define.
 * @MEMBER: Name of the array member.
 * @COUNTER: Name of the __counted_by member.
 * @COUNT: Number of elements in the array; must be compile-time const.
 *
 * Define a zeroed, on-stack, instance of @TYPE structure with a trailing
 * flexible array member.
 * Use __struct_size(@NAME) to get compile-time size of it afterwards.
 */
#define DEFINE_FLEX(TYPE, NAME, MEMBER, COUNTER, COUNT)        \
        _DEFINE_FLEX(TYPE, NAME, MEMBER, COUNT, = { .obj.COUNTER = COUNT, })

#endif /* __LINUX_OVERFLOW_H */











































































   10 






   11 

   10 
   11 




   11 
   11 







    6 

































































    3 



    1 








    4 



















    4 









    4 
    4 















    3 
    3 

    2 


    2 
    1 

    1 

    2 
    2 
    2 







    2 















    2 


    3 




































    3 


    3 


    3 






























    3 




















   13 

   13 


   12 

   12 


   12 


   13 







    8 




    8 


    8 



    8 





    8 

    8 







    5 









    4 
    3 





    3 




    3 






    3 




    3 








    8 







    2 







    5 







    1 



















    5 











    5 




    5 


    4 

    1 






    3 











    3 




    3 


    2 

    1 





   21 
    8 
   13 
    5 
    8 
    5 
    3 
   21 





























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * taskstats.c - Export per-task statistics to userland
 *
 * Copyright (C) Shailabh Nagar, IBM Corp. 2006
 *           (C) Balbir Singh,   IBM Corp. 2006
 */

#include <linux/kernel.h>
#include <linux/taskstats_kern.h>
#include <linux/tsacct_kern.h>
#include <linux/acct.h>
#include <linux/delayacct.h>
#include <linux/cpumask.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/cgroupstats.h>
#include <linux/cgroup.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/pid_namespace.h>
#include <net/genetlink.h>
#include <linux/atomic.h>
#include <linux/sched/cputime.h>

/*
 * Maximum length of a cpumask that can be specified in
 * the TASKSTATS_CMD_ATTR_REGISTER/DEREGISTER_CPUMASK attribute
 */
#define TASKSTATS_CPUMASK_MAXLEN        (100+6*NR_CPUS)

static DEFINE_PER_CPU(__u32, taskstats_seqnum);
static int family_registered;
struct kmem_cache *taskstats_cache;

static struct genl_family family;

static const struct nla_policy taskstats_cmd_get_policy[] = {
        [TASKSTATS_CMD_ATTR_PID]  = { .type = NLA_U32 },
        [TASKSTATS_CMD_ATTR_TGID] = { .type = NLA_U32 },
        [TASKSTATS_CMD_ATTR_REGISTER_CPUMASK] = { .type = NLA_STRING },
        [TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK] = { .type = NLA_STRING },};

static const struct nla_policy cgroupstats_cmd_get_policy[] = {
        [CGROUPSTATS_CMD_ATTR_FD] = { .type = NLA_U32 },
};

struct listener {
        struct list_head list;
        pid_t pid;
        char valid;
};

struct listener_list {
        struct rw_semaphore sem;
        struct list_head list;
};
static DEFINE_PER_CPU(struct listener_list, listener_array);

enum actions {
        REGISTER,
        DEREGISTER,
        CPU_DONT_CARE
};

static int prepare_reply(struct genl_info *info, u8 cmd, struct sk_buff **skbp,
                                size_t size)
{
        struct sk_buff *skb;
        void *reply;

        /*
         * If new attributes are added, please revisit this allocation
         */
        skb = genlmsg_new(size, GFP_KERNEL);
        if (!skb)
                return -ENOMEM;

        if (!info) {
                int seq = this_cpu_inc_return(taskstats_seqnum) - 1;

                reply = genlmsg_put(skb, 0, seq, &family, 0, cmd);
        } else
                reply = genlmsg_put_reply(skb, info, &family, 0, cmd);
        if (reply == NULL) {
                nlmsg_free(skb);
                return -EINVAL;
        }

        *skbp = skb;
        return 0;
}

/*
 * Send taskstats data in @skb to listener with nl_pid @pid
 */
static int send_reply(struct sk_buff *skb, struct genl_info *info)
{
        struct genlmsghdr *genlhdr = nlmsg_data(nlmsg_hdr(skb));
        void *reply = genlmsg_data(genlhdr);

        genlmsg_end(skb, reply);

        return genlmsg_reply(skb, info);
}

/*
 * Send taskstats data in @skb to listeners registered for @cpu's exit data
 */
static void send_cpu_listeners(struct sk_buff *skb,
                                        struct listener_list *listeners)
{
        struct genlmsghdr *genlhdr = nlmsg_data(nlmsg_hdr(skb));
        struct listener *s, *tmp;
        struct sk_buff *skb_next, *skb_cur = skb;
        void *reply = genlmsg_data(genlhdr);
        int delcount = 0;

        genlmsg_end(skb, reply);

        down_read(&listeners->sem);
        list_for_each_entry(s, &listeners->list, list) {
                int rc;

                skb_next = NULL;
                if (!list_is_last(&s->list, &listeners->list)) {
                        skb_next = skb_clone(skb_cur, GFP_KERNEL);
                        if (!skb_next)
                                break;
                }
                rc = genlmsg_unicast(&init_net, skb_cur, s->pid);
                if (rc == -ECONNREFUSED) {
                        s->valid = 0;
                        delcount++;
                }
                skb_cur = skb_next;
        }
        up_read(&listeners->sem);

        if (skb_cur)
                nlmsg_free(skb_cur);

        if (!delcount)
                return;

        /* Delete invalidated entries */
        down_write(&listeners->sem);
        list_for_each_entry_safe(s, tmp, &listeners->list, list) {
                if (!s->valid) {
                        list_del(&s->list);
                        kfree(s);
                }
        }
        up_write(&listeners->sem);
}

static void exe_add_tsk(struct taskstats *stats, struct task_struct *tsk)
{
        /* No idea if I'm allowed to access that here, now. */
        struct file *exe_file = get_task_exe_file(tsk);

        if (exe_file) {
                /* Following cp_new_stat64() in stat.c . */
                stats->ac_exe_dev =
                        huge_encode_dev(exe_file->f_inode->i_sb->s_dev);
                stats->ac_exe_inode = exe_file->f_inode->i_ino;
                fput(exe_file);
        } else {
                stats->ac_exe_dev = 0;
                stats->ac_exe_inode = 0;
        }
}

static void fill_stats(struct user_namespace *user_ns,
                       struct pid_namespace *pid_ns,
                       struct task_struct *tsk, struct taskstats *stats)
{
        memset(stats, 0, sizeof(*stats));
        /*
         * Each accounting subsystem adds calls to its functions to
         * fill in relevant parts of struct taskstsats as follows
         *
         *        per-task-foo(stats, tsk);
         */

        delayacct_add_tsk(stats, tsk);

        /* fill in basic acct fields */
        stats->version = TASKSTATS_VERSION;
        stats->nvcsw = tsk->nvcsw;
        stats->nivcsw = tsk->nivcsw;
        bacct_add_tsk(user_ns, pid_ns, stats, tsk);

        /* fill in extended acct fields */
        xacct_add_tsk(stats, tsk);

        /* add executable info */
        exe_add_tsk(stats, tsk);
}

static int fill_stats_for_pid(pid_t pid, struct taskstats *stats)
{
        struct task_struct *tsk;

        tsk = find_get_task_by_vpid(pid);
        if (!tsk)
                return -ESRCH;
        fill_stats(current_user_ns(), task_active_pid_ns(current), tsk, stats);
        put_task_struct(tsk);
        return 0;
}

static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats)
{
        struct task_struct *tsk, *first;
        unsigned long flags;
        int rc = -ESRCH;
        u64 delta, utime, stime;
        u64 start_time;

        /*
         * Add additional stats from live tasks except zombie thread group
         * leaders who are already counted with the dead tasks
         */
        rcu_read_lock();
        first = find_task_by_vpid(tgid);

        if (!first || !lock_task_sighand(first, &flags))
                goto out;

        if (first->signal->stats)
                memcpy(stats, first->signal->stats, sizeof(*stats));
        else
                memset(stats, 0, sizeof(*stats));

        start_time = ktime_get_ns();
        for_each_thread(first, tsk) {
                if (tsk->exit_state)
                        continue;
                /*
                 * Accounting subsystem can call its functions here to
                 * fill in relevant parts of struct taskstsats as follows
                 *
                 *        per-task-foo(stats, tsk);
                 */
                delayacct_add_tsk(stats, tsk);

                /* calculate task elapsed time in nsec */
                delta = start_time - tsk->start_time;
                /* Convert to micro seconds */
                do_div(delta, NSEC_PER_USEC);
                stats->ac_etime += delta;

                task_cputime(tsk, &utime, &stime);
                stats->ac_utime += div_u64(utime, NSEC_PER_USEC);
                stats->ac_stime += div_u64(stime, NSEC_PER_USEC);

                stats->nvcsw += tsk->nvcsw;
                stats->nivcsw += tsk->nivcsw;
        }

        unlock_task_sighand(first, &flags);
        rc = 0;
out:
        rcu_read_unlock();

        stats->version = TASKSTATS_VERSION;
        /*
         * Accounting subsystems can also add calls here to modify
         * fields of taskstats.
         */
        return rc;
}

static void fill_tgid_exit(struct task_struct *tsk)
{
        unsigned long flags;

        spin_lock_irqsave(&tsk->sighand->siglock, flags);
        if (!tsk->signal->stats)
                goto ret;

        /*
         * Each accounting subsystem calls its functions here to
         * accumalate its per-task stats for tsk, into the per-tgid structure
         *
         *        per-task-foo(tsk->signal->stats, tsk);
         */
        delayacct_add_tsk(tsk->signal->stats, tsk);
ret:
        spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
        return;
}

static int add_del_listener(pid_t pid, const struct cpumask *mask, int isadd)
{
        struct listener_list *listeners;
        struct listener *s, *tmp, *s2;
        unsigned int cpu;
        int ret = 0;

        if (!cpumask_subset(mask, cpu_possible_mask))
                return -EINVAL;

        if (current_user_ns() != &init_user_ns)
                return -EINVAL;

        if (task_active_pid_ns(current) != &init_pid_ns)
                return -EINVAL;

        if (isadd == REGISTER) {
                for_each_cpu(cpu, mask) {
                        s = kmalloc_node(sizeof(struct listener),
                                        GFP_KERNEL, cpu_to_node(cpu));
                        if (!s) {
                                ret = -ENOMEM;
                                goto cleanup;
                        }
                        s->pid = pid;
                        s->valid = 1;

                        listeners = &per_cpu(listener_array, cpu);
                        down_write(&listeners->sem);
                        list_for_each_entry(s2, &listeners->list, list) {
                                if (s2->pid == pid && s2->valid)
                                        goto exists;
                        }
                        list_add(&s->list, &listeners->list);
                        s = NULL;
exists:
                        up_write(&listeners->sem);
                        kfree(s); /* nop if NULL */
                }
                return 0;
        }

        /* Deregister or cleanup */
cleanup:
        for_each_cpu(cpu, mask) {
                listeners = &per_cpu(listener_array, cpu);
                down_write(&listeners->sem);
                list_for_each_entry_safe(s, tmp, &listeners->list, list) {
                        if (s->pid == pid) {
                                list_del(&s->list);
                                kfree(s);
                                break;
                        }
                }
                up_write(&listeners->sem);
        }
        return ret;
}

static int parse(struct nlattr *na, struct cpumask *mask)
{
        char *data;
        int len;
        int ret;

        if (na == NULL)
                return 1;
        len = nla_len(na);
        if (len > TASKSTATS_CPUMASK_MAXLEN)
                return -E2BIG;
        if (len < 1)
                return -EINVAL;
        data = kmalloc(len, GFP_KERNEL);
        if (!data)
                return -ENOMEM;
        nla_strscpy(data, na, len);
        ret = cpulist_parse(data, mask);
        kfree(data);
        return ret;
}

static struct taskstats *mk_reply(struct sk_buff *skb, int type, u32 pid)
{
        struct nlattr *na, *ret;
        int aggr;

        aggr = (type == TASKSTATS_TYPE_PID)
                        ? TASKSTATS_TYPE_AGGR_PID
                        : TASKSTATS_TYPE_AGGR_TGID;

        na = nla_nest_start_noflag(skb, aggr);
        if (!na)
                goto err;

        if (nla_put(skb, type, sizeof(pid), &pid) < 0) {
                nla_nest_cancel(skb, na);
                goto err;
        }
        ret = nla_reserve_64bit(skb, TASKSTATS_TYPE_STATS,
                                sizeof(struct taskstats), TASKSTATS_TYPE_NULL);
        if (!ret) {
                nla_nest_cancel(skb, na);
                goto err;
        }
        nla_nest_end(skb, na);

        return nla_data(ret);
err:
        return NULL;
}

static int cgroupstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
{
        int rc = 0;
        struct sk_buff *rep_skb;
        struct cgroupstats *stats;
        struct nlattr *na;
        size_t size;
        u32 fd;

        na = info->attrs[CGROUPSTATS_CMD_ATTR_FD];
        if (!na)
                return -EINVAL;

        fd = nla_get_u32(info->attrs[CGROUPSTATS_CMD_ATTR_FD]);
        CLASS(fd, f)(fd);
        if (fd_empty(f))
                return 0;

        size = nla_total_size(sizeof(struct cgroupstats));

        rc = prepare_reply(info, CGROUPSTATS_CMD_NEW, &rep_skb,
                                size);
        if (rc < 0)
                return rc;

        na = nla_reserve(rep_skb, CGROUPSTATS_TYPE_CGROUP_STATS,
                                sizeof(struct cgroupstats));
        if (na == NULL) {
                nlmsg_free(rep_skb);
                return -EMSGSIZE;
        }

        stats = nla_data(na);
        memset(stats, 0, sizeof(*stats));

        rc = cgroupstats_build(stats, fd_file(f)->f_path.dentry);
        if (rc < 0) {
                nlmsg_free(rep_skb);
                return rc;
        }

        return send_reply(rep_skb, info);
}

static int cmd_attr_register_cpumask(struct genl_info *info)
{
        cpumask_var_t mask;
        int rc;

        if (!alloc_cpumask_var(&mask, GFP_KERNEL))
                return -ENOMEM;
        rc = parse(info->attrs[TASKSTATS_CMD_ATTR_REGISTER_CPUMASK], mask);
        if (rc < 0)
                goto out;
        rc = add_del_listener(info->snd_portid, mask, REGISTER);
out:
        free_cpumask_var(mask);
        return rc;
}

static int cmd_attr_deregister_cpumask(struct genl_info *info)
{
        cpumask_var_t mask;
        int rc;

        if (!alloc_cpumask_var(&mask, GFP_KERNEL))
                return -ENOMEM;
        rc = parse(info->attrs[TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK], mask);
        if (rc < 0)
                goto out;
        rc = add_del_listener(info->snd_portid, mask, DEREGISTER);
out:
        free_cpumask_var(mask);
        return rc;
}

static size_t taskstats_packet_size(void)
{
        size_t size;

        size = nla_total_size(sizeof(u32)) +
                nla_total_size_64bit(sizeof(struct taskstats)) +
                nla_total_size(0);

        return size;
}

static int cmd_attr_pid(struct genl_info *info)
{
        struct taskstats *stats;
        struct sk_buff *rep_skb;
        size_t size;
        u32 pid;
        int rc;

        size = taskstats_packet_size();

        rc = prepare_reply(info, TASKSTATS_CMD_NEW, &rep_skb, size);
        if (rc < 0)
                return rc;

        rc = -EINVAL;
        pid = nla_get_u32(info->attrs[TASKSTATS_CMD_ATTR_PID]);
        stats = mk_reply(rep_skb, TASKSTATS_TYPE_PID, pid);
        if (!stats)
                goto err;

        rc = fill_stats_for_pid(pid, stats);
        if (rc < 0)
                goto err;
        return send_reply(rep_skb, info);
err:
        nlmsg_free(rep_skb);
        return rc;
}

static int cmd_attr_tgid(struct genl_info *info)
{
        struct taskstats *stats;
        struct sk_buff *rep_skb;
        size_t size;
        u32 tgid;
        int rc;

        size = taskstats_packet_size();

        rc = prepare_reply(info, TASKSTATS_CMD_NEW, &rep_skb, size);
        if (rc < 0)
                return rc;

        rc = -EINVAL;
        tgid = nla_get_u32(info->attrs[TASKSTATS_CMD_ATTR_TGID]);
        stats = mk_reply(rep_skb, TASKSTATS_TYPE_TGID, tgid);
        if (!stats)
                goto err;

        rc = fill_stats_for_tgid(tgid, stats);
        if (rc < 0)
                goto err;
        return send_reply(rep_skb, info);
err:
        nlmsg_free(rep_skb);
        return rc;
}

static int taskstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
{
        if (info->attrs[TASKSTATS_CMD_ATTR_REGISTER_CPUMASK])
                return cmd_attr_register_cpumask(info);
        else if (info->attrs[TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK])
                return cmd_attr_deregister_cpumask(info);
        else if (info->attrs[TASKSTATS_CMD_ATTR_PID])
                return cmd_attr_pid(info);
        else if (info->attrs[TASKSTATS_CMD_ATTR_TGID])
                return cmd_attr_tgid(info);
        else
                return -EINVAL;
}

static struct taskstats *taskstats_tgid_alloc(struct task_struct *tsk)
{
        struct signal_struct *sig = tsk->signal;
        struct taskstats *stats_new, *stats;

        /* Pairs with smp_store_release() below. */
        stats = smp_load_acquire(&sig->stats);
        if (stats || thread_group_empty(tsk))
                return stats;

        /* No problem if kmem_cache_zalloc() fails */
        stats_new = kmem_cache_zalloc(taskstats_cache, GFP_KERNEL);

        spin_lock_irq(&tsk->sighand->siglock);
        stats = sig->stats;
        if (!stats) {
                /*
                 * Pairs with smp_store_release() above and order the
                 * kmem_cache_zalloc().
                 */
                smp_store_release(&sig->stats, stats_new);
                stats = stats_new;
                stats_new = NULL;
        }
        spin_unlock_irq(&tsk->sighand->siglock);

        if (stats_new)
                kmem_cache_free(taskstats_cache, stats_new);

        return stats;
}

/* Send pid data out on exit */
void taskstats_exit(struct task_struct *tsk, int group_dead)
{
        int rc;
        struct listener_list *listeners;
        struct taskstats *stats;
        struct sk_buff *rep_skb;
        size_t size;
        int is_thread_group;

        if (!family_registered)
                return;

        /*
         * Size includes space for nested attributes
         */
        size = taskstats_packet_size();

        is_thread_group = !!taskstats_tgid_alloc(tsk);
        if (is_thread_group) {
                /* PID + STATS + TGID + STATS */
                size = 2 * size;
                /* fill the tsk->signal->stats structure */
                fill_tgid_exit(tsk);
        }

        listeners = raw_cpu_ptr(&listener_array);
        if (list_empty(&listeners->list))
                return;

        rc = prepare_reply(NULL, TASKSTATS_CMD_NEW, &rep_skb, size);
        if (rc < 0)
                return;

        stats = mk_reply(rep_skb, TASKSTATS_TYPE_PID,
                         task_pid_nr_ns(tsk, &init_pid_ns));
        if (!stats)
                goto err;

        fill_stats(&init_user_ns, &init_pid_ns, tsk, stats);
        if (group_dead)
                stats->ac_flag |= AGROUP;

        /*
         * Doesn't matter if tsk is the leader or the last group member leaving
         */
        if (!is_thread_group || !group_dead)
                goto send;

        stats = mk_reply(rep_skb, TASKSTATS_TYPE_TGID,
                         task_tgid_nr_ns(tsk, &init_pid_ns));
        if (!stats)
                goto err;

        memcpy(stats, tsk->signal->stats, sizeof(*stats));

send:
        send_cpu_listeners(rep_skb, listeners);
        return;
err:
        nlmsg_free(rep_skb);
}

static const struct genl_ops taskstats_ops[] = {
        {
                .cmd                = TASKSTATS_CMD_GET,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit                = taskstats_user_cmd,
                .policy                = taskstats_cmd_get_policy,
                .maxattr        = ARRAY_SIZE(taskstats_cmd_get_policy) - 1,
                .flags                = GENL_ADMIN_PERM,
        },
        {
                .cmd                = CGROUPSTATS_CMD_GET,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit                = cgroupstats_user_cmd,
                .policy                = cgroupstats_cmd_get_policy,
                .maxattr        = ARRAY_SIZE(cgroupstats_cmd_get_policy) - 1,
        },
};

static struct genl_family family __ro_after_init = {
        .name                = TASKSTATS_GENL_NAME,
        .version        = TASKSTATS_GENL_VERSION,
        .module                = THIS_MODULE,
        .ops                = taskstats_ops,
        .n_ops                = ARRAY_SIZE(taskstats_ops),
        .resv_start_op        = CGROUPSTATS_CMD_GET + 1,
        .netnsok        = true,
};

/* Needed early in initialization */
void __init taskstats_init_early(void)
{
        unsigned int i;

        taskstats_cache = KMEM_CACHE(taskstats, SLAB_PANIC);
        for_each_possible_cpu(i) {
                INIT_LIST_HEAD(&(per_cpu(listener_array, i).list));
                init_rwsem(&(per_cpu(listener_array, i).sem));
        }
}

static int __init taskstats_init(void)
{
        int rc;

        rc = genl_register_family(&family);
        if (rc)
                return rc;

        family_registered = 1;
        pr_info("registered taskstats version %d\n", TASKSTATS_GENL_VERSION);
        return 0;
}

/*
 * late initcall ensures initialization of statistics collection
 * mechanisms precedes initialization of the taskstats interface
 */
late_initcall(taskstats_init);


































































































    6 
    6 



    6 




    6 
    6 


    5 







    2 











    1 

























































    2 




   15 
   11 
   15 



   15 




   15 
    2 

   13 


   12 


   11 








   11 


   10 



    1 
    1 




    1 
    1 



    1 
    1 



    1 





    1 
    1 




    1 
    1 




    1 
    1 




    1 
    1 












    1 
    1 

    1 
    1 






    9 



    9 













   63 

   59 














































    1 
























    2 













   34 


















   82 










   65 















    1 































   65 


   65 



   64 

   65 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
// SPDX-License-Identifier: GPL-2.0
#include <linux/anon_inodes.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/cgroup.h>
#include <linux/magic.h>
#include <linux/mount.h>
#include <linux/pid.h>
#include <linux/pidfs.h>
#include <linux/pid_namespace.h>
#include <linux/poll.h>
#include <linux/proc_fs.h>
#include <linux/proc_ns.h>
#include <linux/pseudo_fs.h>
#include <linux/ptrace.h>
#include <linux/seq_file.h>
#include <uapi/linux/pidfd.h>
#include <linux/ipc_namespace.h>
#include <linux/time_namespace.h>
#include <linux/utsname.h>
#include <net/net_namespace.h>

#include "internal.h"
#include "mount.h"

#ifdef CONFIG_PROC_FS
/**
 * pidfd_show_fdinfo - print information about a pidfd
 * @m: proc fdinfo file
 * @f: file referencing a pidfd
 *
 * Pid:
 * This function will print the pid that a given pidfd refers to in the
 * pid namespace of the procfs instance.
 * If the pid namespace of the process is not a descendant of the pid
 * namespace of the procfs instance 0 will be shown as its pid. This is
 * similar to calling getppid() on a process whose parent is outside of
 * its pid namespace.
 *
 * NSpid:
 * If pid namespaces are supported then this function will also print
 * the pid of a given pidfd refers to for all descendant pid namespaces
 * starting from the current pid namespace of the instance, i.e. the
 * Pid field and the first entry in the NSpid field will be identical.
 * If the pid namespace of the process is not a descendant of the pid
 * namespace of the procfs instance 0 will be shown as its first NSpid
 * entry and no others will be shown.
 * Note that this differs from the Pid and NSpid fields in
 * /proc/<pid>/status where Pid and NSpid are always shown relative to
 * the  pid namespace of the procfs instance. The difference becomes
 * obvious when sending around a pidfd between pid namespaces from a
 * different branch of the tree, i.e. where no ancestral relation is
 * present between the pid namespaces:
 * - create two new pid namespaces ns1 and ns2 in the initial pid
 *   namespace (also take care to create new mount namespaces in the
 *   new pid namespace and mount procfs)
 * - create a process with a pidfd in ns1
 * - send pidfd from ns1 to ns2
 * - read /proc/self/fdinfo/<pidfd> and observe that both Pid and NSpid
 *   have exactly one entry, which is 0
 */
static void pidfd_show_fdinfo(struct seq_file *m, struct file *f)
{
        struct pid *pid = pidfd_pid(f);
        struct pid_namespace *ns;
        pid_t nr = -1;

        if (likely(pid_has_task(pid, PIDTYPE_PID))) {
                ns = proc_pid_ns(file_inode(m->file)->i_sb);
                nr = pid_nr_ns(pid, ns);
        }

        seq_put_decimal_ll(m, "Pid:\t", nr);

#ifdef CONFIG_PID_NS
        seq_put_decimal_ll(m, "\nNSpid:\t", nr);
        if (nr > 0) {
                int i;

                /* If nr is non-zero it means that 'pid' is valid and that
                 * ns, i.e. the pid namespace associated with the procfs
                 * instance, is in the pid namespace hierarchy of pid.
                 * Start at one below the already printed level.
                 */
                for (i = ns->level + 1; i <= pid->level; i++)
                        seq_put_decimal_ll(m, "\t", pid->numbers[i].nr);
        }
#endif
        seq_putc(m, '\n');
}
#endif

/*
 * Poll support for process exit notification.
 */
static __poll_t pidfd_poll(struct file *file, struct poll_table_struct *pts)
{
        struct pid *pid = pidfd_pid(file);
        bool thread = file->f_flags & PIDFD_THREAD;
        struct task_struct *task;
        __poll_t poll_flags = 0;

        poll_wait(file, &pid->wait_pidfd, pts);
        /*
         * Depending on PIDFD_THREAD, inform pollers when the thread
         * or the whole thread-group exits.
         */
        guard(rcu)();
        task = pid_task(pid, PIDTYPE_PID);
        if (!task)
                poll_flags = EPOLLIN | EPOLLRDNORM | EPOLLHUP;
        else if (task->exit_state && (thread || thread_group_empty(task)))
                poll_flags = EPOLLIN | EPOLLRDNORM;

        return poll_flags;
}

static long pidfd_info(struct task_struct *task, unsigned int cmd, unsigned long arg)
{
        struct pidfd_info __user *uinfo = (struct pidfd_info __user *)arg;
        size_t usize = _IOC_SIZE(cmd);
        struct pidfd_info kinfo = {};
        struct user_namespace *user_ns;
        const struct cred *c;
        __u64 mask;
#ifdef CONFIG_CGROUPS
        struct cgroup *cgrp;
#endif

        if (!uinfo)
                return -EINVAL;
        if (usize < PIDFD_INFO_SIZE_VER0)
                return -EINVAL; /* First version, no smaller struct possible */

        if (copy_from_user(&mask, &uinfo->mask, sizeof(mask)))
                return -EFAULT;

        c = get_task_cred(task);
        if (!c)
                return -ESRCH;

        /* Unconditionally return identifiers and credentials, the rest only on request */

        user_ns = current_user_ns();
        kinfo.ruid = from_kuid_munged(user_ns, c->uid);
        kinfo.rgid = from_kgid_munged(user_ns, c->gid);
        kinfo.euid = from_kuid_munged(user_ns, c->euid);
        kinfo.egid = from_kgid_munged(user_ns, c->egid);
        kinfo.suid = from_kuid_munged(user_ns, c->suid);
        kinfo.sgid = from_kgid_munged(user_ns, c->sgid);
        kinfo.fsuid = from_kuid_munged(user_ns, c->fsuid);
        kinfo.fsgid = from_kgid_munged(user_ns, c->fsgid);
        kinfo.mask |= PIDFD_INFO_CREDS;
        put_cred(c);

#ifdef CONFIG_CGROUPS
        rcu_read_lock();
        cgrp = task_dfl_cgroup(task);
        kinfo.cgroupid = cgroup_id(cgrp);
        kinfo.mask |= PIDFD_INFO_CGROUPID;
        rcu_read_unlock();
#endif

        /*
         * Copy pid/tgid last, to reduce the chances the information might be
         * stale. Note that it is not possible to ensure it will be valid as the
         * task might return as soon as the copy_to_user finishes, but that's ok
         * and userspace expects that might happen and can act accordingly, so
         * this is just best-effort. What we can do however is checking that all
         * the fields are set correctly, or return ESRCH to avoid providing
         * incomplete information. */

        kinfo.ppid = task_ppid_nr_ns(task, NULL);
        kinfo.tgid = task_tgid_vnr(task);
        kinfo.pid = task_pid_vnr(task);
        kinfo.mask |= PIDFD_INFO_PID;

        if (kinfo.pid == 0 || kinfo.tgid == 0 || (kinfo.ppid == 0 && kinfo.pid != 1))
                return -ESRCH;

        /*
         * If userspace and the kernel have the same struct size it can just
         * be copied. If userspace provides an older struct, only the bits that
         * userspace knows about will be copied. If userspace provides a new
         * struct, only the bits that the kernel knows about will be copied.
         */
        if (copy_to_user(uinfo, &kinfo, min(usize, sizeof(kinfo))))
                return -EFAULT;

        return 0;
}

static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
        struct task_struct *task __free(put_task) = NULL;
        struct nsproxy *nsp __free(put_nsproxy) = NULL;
        struct pid *pid = pidfd_pid(file);
        struct ns_common *ns_common = NULL;
        struct pid_namespace *pid_ns;

        task = get_pid_task(pid, PIDTYPE_PID);
        if (!task)
                return -ESRCH;

        /* Extensible IOCTL that does not open namespace FDs, take a shortcut */
        if (_IOC_NR(cmd) == _IOC_NR(PIDFD_GET_INFO))
                return pidfd_info(task, cmd, arg);

        if (arg)
                return -EINVAL;

        scoped_guard(task_lock, task) {
                nsp = task->nsproxy;
                if (nsp)
                        get_nsproxy(nsp);
        }
        if (!nsp)
                return -ESRCH; /* just pretend it didn't exist */

        /*
         * We're trying to open a file descriptor to the namespace so perform a
         * filesystem cred ptrace check. Also, we mirror nsfs behavior.
         */
        if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
                return -EACCES;

        switch (cmd) {
        /* Namespaces that hang of nsproxy. */
        case PIDFD_GET_CGROUP_NAMESPACE:
                if (IS_ENABLED(CONFIG_CGROUPS)) {
                        get_cgroup_ns(nsp->cgroup_ns);
                        ns_common = to_ns_common(nsp->cgroup_ns);
                }
                break;
        case PIDFD_GET_IPC_NAMESPACE:
                if (IS_ENABLED(CONFIG_IPC_NS)) {
                        get_ipc_ns(nsp->ipc_ns);
                        ns_common = to_ns_common(nsp->ipc_ns);
                }
                break;
        case PIDFD_GET_MNT_NAMESPACE:
                get_mnt_ns(nsp->mnt_ns);
                ns_common = to_ns_common(nsp->mnt_ns);
                break;
        case PIDFD_GET_NET_NAMESPACE:
                if (IS_ENABLED(CONFIG_NET_NS)) {
                        ns_common = to_ns_common(nsp->net_ns);
                        get_net_ns(ns_common);
                }
                break;
        case PIDFD_GET_PID_FOR_CHILDREN_NAMESPACE:
                if (IS_ENABLED(CONFIG_PID_NS)) {
                        get_pid_ns(nsp->pid_ns_for_children);
                        ns_common = to_ns_common(nsp->pid_ns_for_children);
                }
                break;
        case PIDFD_GET_TIME_NAMESPACE:
                if (IS_ENABLED(CONFIG_TIME_NS)) {
                        get_time_ns(nsp->time_ns);
                        ns_common = to_ns_common(nsp->time_ns);
                }
                break;
        case PIDFD_GET_TIME_FOR_CHILDREN_NAMESPACE:
                if (IS_ENABLED(CONFIG_TIME_NS)) {
                        get_time_ns(nsp->time_ns_for_children);
                        ns_common = to_ns_common(nsp->time_ns_for_children);
                }
                break;
        case PIDFD_GET_UTS_NAMESPACE:
                if (IS_ENABLED(CONFIG_UTS_NS)) {
                        get_uts_ns(nsp->uts_ns);
                        ns_common = to_ns_common(nsp->uts_ns);
                }
                break;
        /* Namespaces that don't hang of nsproxy. */
        case PIDFD_GET_USER_NAMESPACE:
                if (IS_ENABLED(CONFIG_USER_NS)) {
                        rcu_read_lock();
                        ns_common = to_ns_common(get_user_ns(task_cred_xxx(task, user_ns)));
                        rcu_read_unlock();
                }
                break;
        case PIDFD_GET_PID_NAMESPACE:
                if (IS_ENABLED(CONFIG_PID_NS)) {
                        rcu_read_lock();
                        pid_ns = task_active_pid_ns(task);
                        if (pid_ns)
                                ns_common = to_ns_common(get_pid_ns(pid_ns));
                        rcu_read_unlock();
                }
                break;
        default:
                return -ENOIOCTLCMD;
        }

        if (!ns_common)
                return -EOPNOTSUPP;

        /* open_namespace() unconditionally consumes the reference */
        return open_namespace(ns_common);
}

static const struct file_operations pidfs_file_operations = {
        .poll                = pidfd_poll,
#ifdef CONFIG_PROC_FS
        .show_fdinfo        = pidfd_show_fdinfo,
#endif
        .unlocked_ioctl        = pidfd_ioctl,
        .compat_ioctl   = compat_ptr_ioctl,
};

struct pid *pidfd_pid(const struct file *file)
{
        if (file->f_op != &pidfs_file_operations)
                return ERR_PTR(-EBADF);
        return file_inode(file)->i_private;
}

static struct vfsmount *pidfs_mnt __ro_after_init;

#if BITS_PER_LONG == 32
/*
 * Provide a fallback mechanism for 32-bit systems so processes remain
 * reliably comparable by inode number even on those systems.
 */
static DEFINE_IDA(pidfd_inum_ida);

static int pidfs_inum(struct pid *pid, unsigned long *ino)
{
        int ret;

        ret = ida_alloc_range(&pidfd_inum_ida, RESERVED_PIDS + 1,
                              UINT_MAX, GFP_ATOMIC);
        if (ret < 0)
                return -ENOSPC;

        *ino = ret;
        return 0;
}

static inline void pidfs_free_inum(unsigned long ino)
{
        if (ino > 0)
                ida_free(&pidfd_inum_ida, ino);
}
#else
static inline int pidfs_inum(struct pid *pid, unsigned long *ino)
{
        *ino = pid->ino;
        return 0;
}
#define pidfs_free_inum(ino) ((void)(ino))
#endif

/*
 * The vfs falls back to simple_setattr() if i_op->setattr() isn't
 * implemented. Let's reject it completely until we have a clean
 * permission concept for pidfds.
 */
static int pidfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
                         struct iattr *attr)
{
        return -EOPNOTSUPP;
}


/*
 * User space expects pidfs inodes to have no file type in st_mode.
 *
 * In particular, 'lsof' has this legacy logic:
 *
 *        type = s->st_mode & S_IFMT;
 *        switch (type) {
 *          ...
 *        case 0:
 *                if (!strcmp(p, "anon_inode"))
 *                        Lf->ntype = Ntype = N_ANON_INODE;
 *
 * to detect our old anon_inode logic.
 *
 * Rather than mess with our internal sane inode data, just fix it
 * up here in getattr() by masking off the format bits.
 */
static int pidfs_getattr(struct mnt_idmap *idmap, const struct path *path,
                         struct kstat *stat, u32 request_mask,
                         unsigned int query_flags)
{
        struct inode *inode = d_inode(path->dentry);

        generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat);
        stat->mode &= ~S_IFMT;
        return 0;
}

static const struct inode_operations pidfs_inode_operations = {
        .getattr = pidfs_getattr,
        .setattr = pidfs_setattr,
};

static void pidfs_evict_inode(struct inode *inode)
{
        struct pid *pid = inode->i_private;

        clear_inode(inode);
        put_pid(pid);
        pidfs_free_inum(inode->i_ino);
}

static const struct super_operations pidfs_sops = {
        .drop_inode        = generic_delete_inode,
        .evict_inode        = pidfs_evict_inode,
        .statfs                = simple_statfs,
};

/*
 * 'lsof' has knowledge of out historical anon_inode use, and expects
 * the pidfs dentry name to start with 'anon_inode'.
 */
static char *pidfs_dname(struct dentry *dentry, char *buffer, int buflen)
{
        return dynamic_dname(buffer, buflen, "anon_inode:[pidfd]");
}

static const struct dentry_operations pidfs_dentry_operations = {
        .d_delete        = always_delete_dentry,
        .d_dname        = pidfs_dname,
        .d_prune        = stashed_dentry_prune,
};

static int pidfs_init_inode(struct inode *inode, void *data)
{
        inode->i_private = data;
        inode->i_flags |= S_PRIVATE;
        inode->i_mode |= S_IRWXU;
        inode->i_op = &pidfs_inode_operations;
        inode->i_fop = &pidfs_file_operations;
        /*
         * Inode numbering for pidfs start at RESERVED_PIDS + 1. This
         * avoids collisions with the root inode which is 1 for pseudo
         * filesystems.
         */
        return pidfs_inum(data, &inode->i_ino);
}

static void pidfs_put_data(void *data)
{
        struct pid *pid = data;
        put_pid(pid);
}

static const struct stashed_operations pidfs_stashed_ops = {
        .init_inode = pidfs_init_inode,
        .put_data = pidfs_put_data,
};

static int pidfs_init_fs_context(struct fs_context *fc)
{
        struct pseudo_fs_context *ctx;

        ctx = init_pseudo(fc, PID_FS_MAGIC);
        if (!ctx)
                return -ENOMEM;

        ctx->ops = &pidfs_sops;
        ctx->dops = &pidfs_dentry_operations;
        fc->s_fs_info = (void *)&pidfs_stashed_ops;
        return 0;
}

static struct file_system_type pidfs_type = {
        .name                        = "pidfs",
        .init_fs_context        = pidfs_init_fs_context,
        .kill_sb                = kill_anon_super,
};

struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags)
{

        struct file *pidfd_file;
        struct path path;
        int ret;

        ret = path_from_stashed(&pid->stashed, pidfs_mnt, get_pid(pid), &path);
        if (ret < 0)
                return ERR_PTR(ret);

        pidfd_file = dentry_open(&path, flags, current_cred());
        path_put(&path);
        return pidfd_file;
}

void __init pidfs_init(void)
{
        pidfs_mnt = kern_mount(&pidfs_type);
        if (IS_ERR(pidfs_mnt))
                panic("Failed to mount pidfs pseudo filesystem");
}






































    5 




    4 

    5 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
// SPDX-License-Identifier: GPL-2.0
/*
 * Handling of different ABIs (personalities).
 *
 * We group personalities into execution domains which have their
 * own handlers for kernel entry points, signal mapping, etc...
 *
 * 2001-05-06        Complete rewrite,  Christoph Hellwig (hch@infradead.org)
 */

#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/kmod.h>
#include <linux/module.h>
#include <linux/personality.h>
#include <linux/proc_fs.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/syscalls.h>
#include <linux/sysctl.h>
#include <linux/types.h>

#ifdef CONFIG_PROC_FS
static int execdomains_proc_show(struct seq_file *m, void *v)
{
        seq_puts(m, "0-0\tLinux           \t[kernel]\n");
        return 0;
}

static int __init proc_execdomains_init(void)
{
        proc_create_single("execdomains", 0, NULL, execdomains_proc_show);
        return 0;
}
module_init(proc_execdomains_init);
#endif

SYSCALL_DEFINE1(personality, unsigned int, personality)
{
        unsigned int old = current->personality;

        if (personality != 0xffffffff)
                set_personality(personality);

        return old;
}







































































































































































































































































































































































































































































































































































































































    4 
















































 1397 









 1593 


 1595 





  309 








































  110 
  110 

  110 



    8 

  110 














  104 









   24 





    1 
























 1454 





 1398 






   24 













  182 


































  325 







  323 





















 1469 


 1468 

 1463 
  326 





  760 




  183 




  183 

  183 








  183 



















 1394 

















































































 1388 


 1381 






























































































  541 

  145 
  142 




























    3 
    3 
    3 








    3 













































































 1395 


 1394 

 1395 
 1385 

 1385 




 1383 



















  674 










 1384 



















































































































































































































































































  105 


  678 
  660 

  678 













  104 




  105 

  105 


  103 


  104 





  104 






  105 


  104 



  105 
  105 



  105 











 1410 
 1398 
 1395 











   31 
   31 





    8 








    8 






    8 







    8 

















  679 




  676 

  677 
  667 















 1390 

  681 


 1394 

  676 

  933 



  681 
 1387 










  675 


  674 










  162 


  162 



  162 











  161 


 1393 

















    6 












































































































































   22 
   16 

   22 





   21 






    6 

















   16 


   16 





















   24 


   22 

   24 



















   24 




































  314 


  310 







  234 
  313 



  243 


   24 




   24 



   24 








   24 
   24 
   22 


















    3 

   24 





   24 






   24 






























  311 

  315 




















 1397 





 1398 
 1402 
 1402 




























  625 































 1404 






 1390 


 1401 


 1400 
 1098 
 1400 

  649 


 1401 
 1392 











  471 


  222 

  222 

  222 

   12 



  219 



 1373 









 1394 











 1404 

 1393 


 1393 







 1392 
 1385 
 1376 

 1385 



  170 




 1395 
 1401 




 1484 


 1476 

 1481 



























 1343 

 1338 
 1334 
 1334 



 1344 
 1328 
































































































  324 


  391 
  389 
  390 
  392 







  389 
  123 














  327 
  326 



















  246 




  184 

  248 
  245 
  246 



  248 
  244 
























  206 


  208 


  206 

  208 
  207 

























    4 






    4 
    4 

    4 










    3 

    3 




































































































    3 























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  312 


  311 


   27 

   26 
































































  310 















  131 




  301 






  309 











































   30 
   31 
   28 










   31 
   30 
   31 
    5 

   31 


   31 
   31 

   31 
    1 





   31 
   31 
   31 




   30 
   30 

   31 
   30 

   31 





  312 


  306 


  311 














  311 
















   31 









   30 














   29 
   30 




   30 

   31 




   30 






    1 











    1 











   31 


    1 





    1 


    1 





    1 






    1 



    1 



















    1 
    1 
























   31 
























    8 

    8 
    8 

    8 






    8 
    8 
    8 













    8 
    8 
    8 











  549 
  545 

  236 



  358 

  300 

  300 


  347 
  347 

  130 


  309 


  312 


  311 










  312 
  281 

  312 


  345 
  535 





  549 




  548 


  549 










    5 


    5 





















  307 


  309 
  546 















  314 
    4 

















  111 
  110 
   19 
  110 














    4 
    4 
    4 


    4 








  183 
  170 







  171 






  180 



  183 




  183 

  183 

  183 
  183 






  170 




  169 





  169 
  171 

  170 
  170 

  170 































   98 





















   97 














   92 




















































  170 


  169 

  169 
  171 
  171 

  171 

  170 



















































































































  105 















  104 







  105 










  105 






























  103 




  105 





  104 




  105 

  104 

  105 


























    3 

    3 
    3 





    3 
    3 



























































  105 




  104 


  105 














































  105 














  105 
  104 


  105 




  105 


  105 
































































































































  105 



  105 


  105 
  105 
  105 



































































































  104 

  105 















  105 

  105 


  105 



  105 


  105 









  105 

  105 



  104 





  105 
  105 




  104 




  105 


















    3 






    3 








  105 


  105 
  105 


  104 





















  105 
  104 
  104 

  105 

  104 













  105 

  105 



  105 

  105 








  105 

  104 




  105 
  105 
  103 


    3 







  103 










  103 


  105 
  105 













  105 




  105 
  105 

  105 









  104 








  105 


  104 




  105 
  105 

  105 







































































































  105 

  105 



  105 




























  104 


  103 


  103 



    3 





















  103 













  104 


  105 

  104 
  105 

    3 







    3 

    3 








    3 

    3 


    3 

















  105 

  105 



  105 



  105 















































  105 







  105 



  105 
  105 








  105 




  104 


    1 


  104 






  104 
  103 



  105 














  105 







  104 

  105 


  104 



  104 

  105 


  105 

  105 
































  105 







  105 































    8 
    8 


    8 

    8 




















    8 


    8 











    3 












    8 


    8 
    8 









    8 

    8 





    8 







    8 

    8 
    8 
    8 


    8 
    8 

    8 










































































    1 

    1 


































































































































    3 







    3 


























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    8 





















































































































































































  946 




  943 




  946 
   79 








































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127
7128
7129
7130
7131
7132
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
7219
7220
7221
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231
7232
7233
7234
7235
7236
7237
7238
7239
7240
7241
7242
7243
7244
7245
7246
7247
7248
7249
7250
7251
7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317
7318
7319
7320
7321
7322
7323
7324
7325
7326
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342
7343
7344
7345
7346
7347
7348
7349
7350
7351
7352
7353
7354
7355
7356
7357
7358
7359
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385
7386
7387
7388
7389
7390
7391
7392
7393
7394
7395
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412
7413
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
7441
7442
7443
7444
7445
7446
7447
7448
7449
7450
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
7466
7467
7468
7469
7470
7471
7472
7473
7474
7475
7476
7477
7478
7479
7480
7481
7482
7483
7484
7485
7486
7487
7488
7489
7490
7491
7492
7493
7494
7495
7496
7497
7498
7499
7500
7501
7502
7503
7504
7505
7506
7507
7508
7509
7510
7511
7512
7513
7514
7515
7516
7517
7518
7519
7520
7521
7522
7523
7524
7525
7526
7527
7528
7529
7530
7531
7532
7533
7534
7535
7536
7537
7538
7539
7540
7541
7542
7543
7544
7545
7546
7547
7548
7549
7550
7551
7552
7553
7554
7555
7556
7557
7558
7559
7560
7561
7562
7563
7564
7565
7566
7567
7568
7569
7570
7571
7572
7573
7574
7575
7576
7577
7578
7579
7580
7581
7582
7583
7584
7585
7586
7587
7588
7589
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599
7600
7601
7602
7603
7604
7605
7606
7607
7608
7609
7610
7611
7612
7613
7614
7615
7616
7617
7618
7619
7620
7621
7622
7623
7624
7625
7626
7627
7628
7629
7630
7631
7632
7633
7634
7635
7636
7637
7638
7639
7640
7641
7642
7643
7644
7645
7646
7647
7648
7649
7650
7651
7652
7653
7654
7655
7656
7657
7658
7659
7660
7661
7662
7663
7664
7665
7666
7667
7668
7669
7670
7671
7672
7673
7674
7675
7676
7677
7678
7679
7680
7681
7682
7683
7684
7685
7686
7687
7688
7689
7690
7691
7692
7693
7694
7695
7696
7697
7698
7699
7700
7701
7702
7703
7704
7705
7706
7707
7708
7709
7710
7711
7712
7713
7714
7715
7716
7717
7718
7719
7720
7721
7722
7723
7724
7725
7726
7727
7728
7729
7730
7731
7732
7733
7734
7735
7736
7737
7738
7739
7740
7741
7742
7743
7744
7745
7746
7747
7748
7749
7750
7751
7752
7753
7754
7755
7756
7757
7758
7759
7760
7761
7762
7763
7764
7765
7766
7767
7768
7769
7770
7771
7772
7773
7774
7775
7776
7777
7778
7779
7780
7781
7782
7783
7784
7785
7786
7787
7788
7789
7790
7791
7792
7793
7794
7795
7796
7797
7798
7799
7800
7801
7802
7803
7804
7805
7806
7807
7808
7809
7810
7811
7812
7813
7814
7815
7816
7817
7818
7819
7820
7821
7822
7823
7824
7825
7826
7827
7828
7829
7830
7831
7832
7833
7834
7835
7836
7837
7838
7839
7840
7841
7842
7843
7844
7845
7846
7847
7848
7849
7850
7851
7852
7853
7854
7855
7856
7857
7858
7859
7860
7861
7862
7863
7864
7865
7866
7867
7868
7869
7870
7871
7872
7873
7874
7875
7876
7877
7878
7879
7880
7881
7882
7883
7884
7885
7886
7887
7888
7889
7890
7891
7892
7893
7894
7895
7896
7897
7898
7899
7900
7901
7902
7903
7904
7905
7906
7907
7908
7909
7910
7911
7912
7913
7914
7915
7916
7917
7918
7919
7920
7921
7922
7923
7924
7925
7926
7927
7928
7929
7930
7931
7932
7933
7934
7935
7936
7937
7938
7939
7940
7941
7942
7943
7944
7945
7946
7947
7948
7949
7950
7951
7952
7953
7954
7955
7956
7957
7958
7959
7960
7961
7962
7963
7964
7965
7966
7967
7968
7969
7970
7971
7972
7973
7974
7975
7976
7977
7978
7979
7980
7981
7982
7983
7984
7985
7986
7987
7988
7989
7990
7991
7992
7993
7994
7995
7996
7997
7998
7999
8000
8001
8002
8003
8004
8005
8006
8007
8008
8009
8010
8011
8012
8013
8014
8015
8016
8017
8018
8019
8020
8021
8022
8023
8024
8025
8026
8027
8028
8029
8030
8031
8032
8033
8034
8035
8036
8037
8038
8039
8040
8041
8042
8043
8044
// SPDX-License-Identifier: GPL-2.0-only
/*
 * kernel/workqueue.c - generic async execution with shared worker pool
 *
 * Copyright (C) 2002                Ingo Molnar
 *
 *   Derived from the taskqueue/keventd code by:
 *     David Woodhouse <dwmw2@infradead.org>
 *     Andrew Morton
 *     Kai Petzke <wpp@marie.physik.tu-berlin.de>
 *     Theodore Ts'o <tytso@mit.edu>
 *
 * Made to use alloc_percpu by Christoph Lameter.
 *
 * Copyright (C) 2010                SUSE Linux Products GmbH
 * Copyright (C) 2010                Tejun Heo <tj@kernel.org>
 *
 * This is the generic async execution mechanism.  Work items as are
 * executed in process context.  The worker pool is shared and
 * automatically managed.  There are two worker pools for each CPU (one for
 * normal work items and the other for high priority ones) and some extra
 * pools for workqueues which are not bound to any specific CPU - the
 * number of these backing pools is dynamic.
 *
 * Please read Documentation/core-api/workqueue.rst for details.
 */

#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/signal.h>
#include <linux/completion.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
#include <linux/hardirq.h>
#include <linux/mempolicy.h>
#include <linux/freezer.h>
#include <linux/debug_locks.h>
#include <linux/lockdep.h>
#include <linux/idr.h>
#include <linux/jhash.h>
#include <linux/hashtable.h>
#include <linux/rculist.h>
#include <linux/nodemask.h>
#include <linux/moduleparam.h>
#include <linux/uaccess.h>
#include <linux/sched/isolation.h>
#include <linux/sched/debug.h>
#include <linux/nmi.h>
#include <linux/kvm_para.h>
#include <linux/delay.h>
#include <linux/irq_work.h>

#include "workqueue_internal.h"

enum worker_pool_flags {
        /*
         * worker_pool flags
         *
         * A bound pool is either associated or disassociated with its CPU.
         * While associated (!DISASSOCIATED), all workers are bound to the
         * CPU and none has %WORKER_UNBOUND set and concurrency management
         * is in effect.
         *
         * While DISASSOCIATED, the cpu may be offline and all workers have
         * %WORKER_UNBOUND set and concurrency management disabled, and may
         * be executing on any CPU.  The pool behaves as an unbound one.
         *
         * Note that DISASSOCIATED should be flipped only while holding
         * wq_pool_attach_mutex to avoid changing binding state while
         * worker_attach_to_pool() is in progress.
         *
         * As there can only be one concurrent BH execution context per CPU, a
         * BH pool is per-CPU and always DISASSOCIATED.
         */
        POOL_BH                        = 1 << 0,        /* is a BH pool */
        POOL_MANAGER_ACTIVE        = 1 << 1,        /* being managed */
        POOL_DISASSOCIATED        = 1 << 2,        /* cpu can't serve workers */
        POOL_BH_DRAINING        = 1 << 3,        /* draining after CPU offline */
};

enum worker_flags {
        /* worker flags */
        WORKER_DIE                = 1 << 1,        /* die die die */
        WORKER_IDLE                = 1 << 2,        /* is idle */
        WORKER_PREP                = 1 << 3,        /* preparing to run works */
        WORKER_CPU_INTENSIVE        = 1 << 6,        /* cpu intensive */
        WORKER_UNBOUND                = 1 << 7,        /* worker is unbound */
        WORKER_REBOUND                = 1 << 8,        /* worker was rebound */

        WORKER_NOT_RUNNING        = WORKER_PREP | WORKER_CPU_INTENSIVE |
                                  WORKER_UNBOUND | WORKER_REBOUND,
};

enum work_cancel_flags {
        WORK_CANCEL_DELAYED        = 1 << 0,        /* canceling a delayed_work */
        WORK_CANCEL_DISABLE        = 1 << 1,        /* canceling to disable */
};

enum wq_internal_consts {
        NR_STD_WORKER_POOLS        = 2,                /* # standard pools per cpu */

        UNBOUND_POOL_HASH_ORDER        = 6,                /* hashed by pool->attrs */
        BUSY_WORKER_HASH_ORDER        = 6,                /* 64 pointers */

        MAX_IDLE_WORKERS_RATIO        = 4,                /* 1/4 of busy can be idle */
        IDLE_WORKER_TIMEOUT        = 300 * HZ,        /* keep idle ones for 5 mins */

        MAYDAY_INITIAL_TIMEOUT  = HZ / 100 >= 2 ? HZ / 100 : 2,
                                                /* call for help after 10ms
                                                   (min two ticks) */
        MAYDAY_INTERVAL                = HZ / 10,        /* and then every 100ms */
        CREATE_COOLDOWN                = HZ,                /* time to breath after fail */

        /*
         * Rescue workers are used only on emergencies and shared by
         * all cpus.  Give MIN_NICE.
         */
        RESCUER_NICE_LEVEL        = MIN_NICE,
        HIGHPRI_NICE_LEVEL        = MIN_NICE,

        WQ_NAME_LEN                = 32,
        WORKER_ID_LEN                = 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */
};

/*
 * We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and
 * MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because
 * msecs_to_jiffies() can't be an initializer.
 */
#define BH_WORKER_JIFFIES        msecs_to_jiffies(2)
#define BH_WORKER_RESTARTS        10

/*
 * Structure fields follow one of the following exclusion rules.
 *
 * I: Modifiable by initialization/destruction paths and read-only for
 *    everyone else.
 *
 * P: Preemption protected.  Disabling preemption is enough and should
 *    only be modified and accessed from the local cpu.
 *
 * L: pool->lock protected.  Access with pool->lock held.
 *
 * LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for
 *     reads.
 *
 * K: Only modified by worker while holding pool->lock. Can be safely read by
 *    self, while holding pool->lock or from IRQ context if %current is the
 *    kworker.
 *
 * S: Only modified by worker self.
 *
 * A: wq_pool_attach_mutex protected.
 *
 * PL: wq_pool_mutex protected.
 *
 * PR: wq_pool_mutex protected for writes.  RCU protected for reads.
 *
 * PW: wq_pool_mutex and wq->mutex protected for writes.  Either for reads.
 *
 * PWR: wq_pool_mutex and wq->mutex protected for writes.  Either or
 *      RCU for reads.
 *
 * WQ: wq->mutex protected.
 *
 * WR: wq->mutex protected for writes.  RCU protected for reads.
 *
 * WO: wq->mutex protected for writes. Updated with WRITE_ONCE() and can be read
 *     with READ_ONCE() without locking.
 *
 * MD: wq_mayday_lock protected.
 *
 * WD: Used internally by the watchdog.
 */

/* struct worker is defined in workqueue_internal.h */

struct worker_pool {
        raw_spinlock_t                lock;                /* the pool lock */
        int                        cpu;                /* I: the associated cpu */
        int                        node;                /* I: the associated node ID */
        int                        id;                /* I: pool ID */
        unsigned int                flags;                /* L: flags */

        unsigned long                watchdog_ts;        /* L: watchdog timestamp */
        bool                        cpu_stall;        /* WD: stalled cpu bound pool */

        /*
         * The counter is incremented in a process context on the associated CPU
         * w/ preemption disabled, and decremented or reset in the same context
         * but w/ pool->lock held. The readers grab pool->lock and are
         * guaranteed to see if the counter reached zero.
         */
        int                        nr_running;

        struct list_head        worklist;        /* L: list of pending works */

        int                        nr_workers;        /* L: total number of workers */
        int                        nr_idle;        /* L: currently idle workers */

        struct list_head        idle_list;        /* L: list of idle workers */
        struct timer_list        idle_timer;        /* L: worker idle timeout */
        struct work_struct      idle_cull_work; /* L: worker idle cleanup */

        struct timer_list        mayday_timer;          /* L: SOS timer for workers */

        /* a workers is either on busy_hash or idle_list, or the manager */
        DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
                                                /* L: hash of busy workers */

        struct worker                *manager;        /* L: purely informational */
        struct list_head        workers;        /* A: attached workers */

        struct ida                worker_ida;        /* worker IDs for task name */

        struct workqueue_attrs        *attrs;                /* I: worker attributes */
        struct hlist_node        hash_node;        /* PL: unbound_pool_hash node */
        int                        refcnt;                /* PL: refcnt for unbound pools */

        /*
         * Destruction of pool is RCU protected to allow dereferences
         * from get_work_pool().
         */
        struct rcu_head                rcu;
};

/*
 * Per-pool_workqueue statistics. These can be monitored using
 * tools/workqueue/wq_monitor.py.
 */
enum pool_workqueue_stats {
        PWQ_STAT_STARTED,        /* work items started execution */
        PWQ_STAT_COMPLETED,        /* work items completed execution */
        PWQ_STAT_CPU_TIME,        /* total CPU time consumed */
        PWQ_STAT_CPU_INTENSIVE,        /* wq_cpu_intensive_thresh_us violations */
        PWQ_STAT_CM_WAKEUP,        /* concurrency-management worker wakeups */
        PWQ_STAT_REPATRIATED,        /* unbound workers brought back into scope */
        PWQ_STAT_MAYDAY,        /* maydays to rescuer */
        PWQ_STAT_RESCUED,        /* linked work items executed by rescuer */

        PWQ_NR_STATS,
};

/*
 * The per-pool workqueue.  While queued, bits below WORK_PWQ_SHIFT
 * of work_struct->data are used for flags and the remaining high bits
 * point to the pwq; thus, pwqs need to be aligned at two's power of the
 * number of flag bits.
 */
struct pool_workqueue {
        struct worker_pool        *pool;                /* I: the associated pool */
        struct workqueue_struct *wq;                /* I: the owning workqueue */
        int                        work_color;        /* L: current color */
        int                        flush_color;        /* L: flushing color */
        int                        refcnt;                /* L: reference count */
        int                        nr_in_flight[WORK_NR_COLORS];
                                                /* L: nr of in_flight works */
        bool                        plugged;        /* L: execution suspended */

        /*
         * nr_active management and WORK_STRUCT_INACTIVE:
         *
         * When pwq->nr_active >= max_active, new work item is queued to
         * pwq->inactive_works instead of pool->worklist and marked with
         * WORK_STRUCT_INACTIVE.
         *
         * All work items marked with WORK_STRUCT_INACTIVE do not participate in
         * nr_active and all work items in pwq->inactive_works are marked with
         * WORK_STRUCT_INACTIVE. But not all WORK_STRUCT_INACTIVE work items are
         * in pwq->inactive_works. Some of them are ready to run in
         * pool->worklist or worker->scheduled. Those work itmes are only struct
         * wq_barrier which is used for flush_work() and should not participate
         * in nr_active. For non-barrier work item, it is marked with
         * WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
         */
        int                        nr_active;        /* L: nr of active works */
        struct list_head        inactive_works;        /* L: inactive works */
        struct list_head        pending_node;        /* LN: node on wq_node_nr_active->pending_pwqs */
        struct list_head        pwqs_node;        /* WR: node on wq->pwqs */
        struct list_head        mayday_node;        /* MD: node on wq->maydays */

        u64                        stats[PWQ_NR_STATS];

        /*
         * Release of unbound pwq is punted to a kthread_worker. See put_pwq()
         * and pwq_release_workfn() for details. pool_workqueue itself is also
         * RCU protected so that the first pwq can be determined without
         * grabbing wq->mutex.
         */
        struct kthread_work        release_work;
        struct rcu_head                rcu;
} __aligned(1 << WORK_STRUCT_PWQ_SHIFT);

/*
 * Structure used to wait for workqueue flush.
 */
struct wq_flusher {
        struct list_head        list;                /* WQ: list of flushers */
        int                        flush_color;        /* WQ: flush color waiting for */
        struct completion        done;                /* flush completion */
};

struct wq_device;

/*
 * Unlike in a per-cpu workqueue where max_active limits its concurrency level
 * on each CPU, in an unbound workqueue, max_active applies to the whole system.
 * As sharing a single nr_active across multiple sockets can be very expensive,
 * the counting and enforcement is per NUMA node.
 *
 * The following struct is used to enforce per-node max_active. When a pwq wants
 * to start executing a work item, it should increment ->nr using
 * tryinc_node_nr_active(). If acquisition fails due to ->nr already being over
 * ->max, the pwq is queued on ->pending_pwqs. As in-flight work items finish
 * and decrement ->nr, node_activate_pending_pwq() activates the pending pwqs in
 * round-robin order.
 */
struct wq_node_nr_active {
        int                        max;                /* per-node max_active */
        atomic_t                nr;                /* per-node nr_active */
        raw_spinlock_t                lock;                /* nests inside pool locks */
        struct list_head        pending_pwqs;        /* LN: pwqs with inactive works */
};

/*
 * The externally visible workqueue.  It relays the issued work items to
 * the appropriate worker_pool through its pool_workqueues.
 */
struct workqueue_struct {
        struct list_head        pwqs;                /* WR: all pwqs of this wq */
        struct list_head        list;                /* PR: list of all workqueues */

        struct mutex                mutex;                /* protects this wq */
        int                        work_color;        /* WQ: current work color */
        int                        flush_color;        /* WQ: current flush color */
        atomic_t                nr_pwqs_to_flush; /* flush in progress */
        struct wq_flusher        *first_flusher;        /* WQ: first flusher */
        struct list_head        flusher_queue;        /* WQ: flush waiters */
        struct list_head        flusher_overflow; /* WQ: flush overflow list */

        struct list_head        maydays;        /* MD: pwqs requesting rescue */
        struct worker                *rescuer;        /* MD: rescue worker */

        int                        nr_drainers;        /* WQ: drain in progress */

        /* See alloc_workqueue() function comment for info on min/max_active */
        int                        max_active;        /* WO: max active works */
        int                        min_active;        /* WO: min active works */
        int                        saved_max_active; /* WQ: saved max_active */
        int                        saved_min_active; /* WQ: saved min_active */

        struct workqueue_attrs        *unbound_attrs;        /* PW: only for unbound wqs */
        struct pool_workqueue __rcu *dfl_pwq;   /* PW: only for unbound wqs */

#ifdef CONFIG_SYSFS
        struct wq_device        *wq_dev;        /* I: for sysfs interface */
#endif
#ifdef CONFIG_LOCKDEP
        char                        *lock_name;
        struct lock_class_key        key;
        struct lockdep_map        __lockdep_map;
        struct lockdep_map        *lockdep_map;
#endif
        char                        name[WQ_NAME_LEN]; /* I: workqueue name */

        /*
         * Destruction of workqueue_struct is RCU protected to allow walking
         * the workqueues list without grabbing wq_pool_mutex.
         * This is used to dump all workqueues from sysrq.
         */
        struct rcu_head                rcu;

        /* hot fields used during command issue, aligned to cacheline */
        unsigned int                flags ____cacheline_aligned; /* WQ: WQ_* flags */
        struct pool_workqueue __rcu * __percpu *cpu_pwq; /* I: per-cpu pwqs */
        struct wq_node_nr_active *node_nr_active[]; /* I: per-node nr_active */
};

/*
 * Each pod type describes how CPUs should be grouped for unbound workqueues.
 * See the comment above workqueue_attrs->affn_scope.
 */
struct wq_pod_type {
        int                        nr_pods;        /* number of pods */
        cpumask_var_t                *pod_cpus;        /* pod -> cpus */
        int                        *pod_node;        /* pod -> node */
        int                        *cpu_pod;        /* cpu -> pod */
};

struct work_offq_data {
        u32                        pool_id;
        u32                        disable;
        u32                        flags;
};

static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = {
        [WQ_AFFN_DFL]                = "default",
        [WQ_AFFN_CPU]                = "cpu",
        [WQ_AFFN_SMT]                = "smt",
        [WQ_AFFN_CACHE]                = "cache",
        [WQ_AFFN_NUMA]                = "numa",
        [WQ_AFFN_SYSTEM]        = "system",
};

/*
 * Per-cpu work items which run for longer than the following threshold are
 * automatically considered CPU intensive and excluded from concurrency
 * management to prevent them from noticeably delaying other per-cpu work items.
 * ULONG_MAX indicates that the user hasn't overridden it with a boot parameter.
 * The actual value is initialized in wq_cpu_intensive_thresh_init().
 */
static unsigned long wq_cpu_intensive_thresh_us = ULONG_MAX;
module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644);
#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
static unsigned int wq_cpu_intensive_warning_thresh = 4;
module_param_named(cpu_intensive_warning_thresh, wq_cpu_intensive_warning_thresh, uint, 0644);
#endif

/* see the comment above the definition of WQ_POWER_EFFICIENT */
static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
module_param_named(power_efficient, wq_power_efficient, bool, 0444);

static bool wq_online;                        /* can kworkers be created yet? */
static bool wq_topo_initialized __read_mostly = false;

static struct kmem_cache *pwq_cache;

static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES];
static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE;

/* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */
static struct workqueue_attrs *unbound_wq_update_pwq_attrs_buf;

static DEFINE_MUTEX(wq_pool_mutex);        /* protects pools and workqueues list */
static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
static DEFINE_RAW_SPINLOCK(wq_mayday_lock);        /* protects wq->maydays list */
/* wait for manager to go away */
static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);

static LIST_HEAD(workqueues);                /* PR: list of all workqueues */
static bool workqueue_freezing;                /* PL: have wqs started freezing? */

/* PL: mirror the cpu_online_mask excluding the CPU in the midst of hotplugging */
static cpumask_var_t wq_online_cpumask;

/* PL&A: allowable cpus for unbound wqs and work items */
static cpumask_var_t wq_unbound_cpumask;

/* PL: user requested unbound cpumask via sysfs */
static cpumask_var_t wq_requested_unbound_cpumask;

/* PL: isolated cpumask to be excluded from unbound cpumask */
static cpumask_var_t wq_isolated_cpumask;

/* for further constrain wq_unbound_cpumask by cmdline parameter*/
static struct cpumask wq_cmdline_cpumask __initdata;

/* CPU where unbound work was last round robin scheduled from this CPU */
static DEFINE_PER_CPU(int, wq_rr_cpu_last);

/*
 * Local execution of unbound work items is no longer guaranteed.  The
 * following always forces round-robin CPU selection on unbound work items
 * to uncover usages which depend on it.
 */
#ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
static bool wq_debug_force_rr_cpu = true;
#else
static bool wq_debug_force_rr_cpu = false;
#endif
module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);

/* to raise softirq for the BH worker pools on other CPUs */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_work [NR_STD_WORKER_POOLS], bh_pool_irq_works);

/* the BH worker pools */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], bh_worker_pools);

/* the per-cpu worker pools */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);

static DEFINE_IDR(worker_pool_idr);        /* PR: idr of all pools */

/* PL: hash of all unbound pools keyed by pool->attrs */
static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);

/* I: attributes used when instantiating standard unbound pools on demand */
static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];

/* I: attributes used when instantiating ordered pools on demand */
static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];

/*
 * I: kthread_worker to release pwq's. pwq release needs to be bounced to a
 * process context while holding a pool lock. Bounce to a dedicated kthread
 * worker to avoid A-A deadlocks.
 */
static struct kthread_worker *pwq_release_worker __ro_after_init;

struct workqueue_struct *system_wq __ro_after_init;
EXPORT_SYMBOL(system_wq);
struct workqueue_struct *system_highpri_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_highpri_wq);
struct workqueue_struct *system_long_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_long_wq);
struct workqueue_struct *system_unbound_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_unbound_wq);
struct workqueue_struct *system_freezable_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_freezable_wq);
struct workqueue_struct *system_power_efficient_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_power_efficient_wq);
struct workqueue_struct *system_freezable_power_efficient_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
struct workqueue_struct *system_bh_wq;
EXPORT_SYMBOL_GPL(system_bh_wq);
struct workqueue_struct *system_bh_highpri_wq;
EXPORT_SYMBOL_GPL(system_bh_highpri_wq);

static int worker_thread(void *__worker);
static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
static void show_pwq(struct pool_workqueue *pwq);
static void show_one_worker_pool(struct worker_pool *pool);

#define CREATE_TRACE_POINTS
#include <trace/events/workqueue.h>

#define assert_rcu_or_pool_mutex()                                        \
        RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() &&                        \
                         !lockdep_is_held(&wq_pool_mutex),                \
                         "RCU or wq_pool_mutex should be held")

#define assert_rcu_or_wq_mutex_or_pool_mutex(wq)                        \
        RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() &&                        \
                         !lockdep_is_held(&wq->mutex) &&                \
                         !lockdep_is_held(&wq_pool_mutex),                \
                         "RCU, wq->mutex or wq_pool_mutex should be held")

#define for_each_bh_worker_pool(pool, cpu)                                \
        for ((pool) = &per_cpu(bh_worker_pools, cpu)[0];                \
             (pool) < &per_cpu(bh_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
             (pool)++)

#define for_each_cpu_worker_pool(pool, cpu)                                \
        for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];                \
             (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
             (pool)++)

/**
 * for_each_pool - iterate through all worker_pools in the system
 * @pool: iteration cursor
 * @pi: integer used for iteration
 *
 * This must be called either with wq_pool_mutex held or RCU read
 * locked.  If the pool needs to be used beyond the locking in effect, the
 * caller is responsible for guaranteeing that the pool stays online.
 *
 * The if/else clause exists only for the lockdep assertion and can be
 * ignored.
 */
#define for_each_pool(pool, pi)                                                \
        idr_for_each_entry(&worker_pool_idr, pool, pi)                        \
                if (({ assert_rcu_or_pool_mutex(); false; })) { }        \
                else

/**
 * for_each_pool_worker - iterate through all workers of a worker_pool
 * @worker: iteration cursor
 * @pool: worker_pool to iterate workers of
 *
 * This must be called with wq_pool_attach_mutex.
 *
 * The if/else clause exists only for the lockdep assertion and can be
 * ignored.
 */
#define for_each_pool_worker(worker, pool)                                \
        list_for_each_entry((worker), &(pool)->workers, node)                \
                if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
                else

/**
 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
 * @pwq: iteration cursor
 * @wq: the target workqueue
 *
 * This must be called either with wq->mutex held or RCU read locked.
 * If the pwq needs to be used beyond the locking in effect, the caller is
 * responsible for guaranteeing that the pwq stays online.
 *
 * The if/else clause exists only for the lockdep assertion and can be
 * ignored.
 */
#define for_each_pwq(pwq, wq)                                                \
        list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node,                \
                                 lockdep_is_held(&(wq->mutex)))

#ifdef CONFIG_DEBUG_OBJECTS_WORK

static const struct debug_obj_descr work_debug_descr;

static void *work_debug_hint(void *addr)
{
        return ((struct work_struct *) addr)->func;
}

static bool work_is_static_object(void *addr)
{
        struct work_struct *work = addr;

        return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
}

/*
 * fixup_init is called when:
 * - an active object is initialized
 */
static bool work_fixup_init(void *addr, enum debug_obj_state state)
{
        struct work_struct *work = addr;

        switch (state) {
        case ODEBUG_STATE_ACTIVE:
                cancel_work_sync(work);
                debug_object_init(work, &work_debug_descr);
                return true;
        default:
                return false;
        }
}

/*
 * fixup_free is called when:
 * - an active object is freed
 */
static bool work_fixup_free(void *addr, enum debug_obj_state state)
{
        struct work_struct *work = addr;

        switch (state) {
        case ODEBUG_STATE_ACTIVE:
                cancel_work_sync(work);
                debug_object_free(work, &work_debug_descr);
                return true;
        default:
                return false;
        }
}

static const struct debug_obj_descr work_debug_descr = {
        .name                = "work_struct",
        .debug_hint        = work_debug_hint,
        .is_static_object = work_is_static_object,
        .fixup_init        = work_fixup_init,
        .fixup_free        = work_fixup_free,
};

static inline void debug_work_activate(struct work_struct *work)
{
        debug_object_activate(work, &work_debug_descr);
}

static inline void debug_work_deactivate(struct work_struct *work)
{
        debug_object_deactivate(work, &work_debug_descr);
}

void __init_work(struct work_struct *work, int onstack)
{
        if (onstack)
                debug_object_init_on_stack(work, &work_debug_descr);
        else
                debug_object_init(work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(__init_work);

void destroy_work_on_stack(struct work_struct *work)
{
        debug_object_free(work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_work_on_stack);

void destroy_delayed_work_on_stack(struct delayed_work *work)
{
        destroy_timer_on_stack(&work->timer);
        debug_object_free(&work->work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);

#else
static inline void debug_work_activate(struct work_struct *work) { }
static inline void debug_work_deactivate(struct work_struct *work) { }
#endif

/**
 * worker_pool_assign_id - allocate ID and assign it to @pool
 * @pool: the pool pointer of interest
 *
 * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
 * successfully, -errno on failure.
 */
static int worker_pool_assign_id(struct worker_pool *pool)
{
        int ret;

        lockdep_assert_held(&wq_pool_mutex);

        ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
                        GFP_KERNEL);
        if (ret >= 0) {
                pool->id = ret;
                return 0;
        }
        return ret;
}

static struct pool_workqueue __rcu **
unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
{
       if (cpu >= 0)
               return per_cpu_ptr(wq->cpu_pwq, cpu);
       else
               return &wq->dfl_pwq;
}

/* @cpu < 0 for dfl_pwq */
static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
{
        return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
                                     lockdep_is_held(&wq_pool_mutex) ||
                                     lockdep_is_held(&wq->mutex));
}

/**
 * unbound_effective_cpumask - effective cpumask of an unbound workqueue
 * @wq: workqueue of interest
 *
 * @wq->unbound_attrs->cpumask contains the cpumask requested by the user which
 * is masked with wq_unbound_cpumask to determine the effective cpumask. The
 * default pwq is always mapped to the pool with the current effective cpumask.
 */
static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq)
{
        return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask;
}

static unsigned int work_color_to_flags(int color)
{
        return color << WORK_STRUCT_COLOR_SHIFT;
}

static int get_work_color(unsigned long work_data)
{
        return (work_data >> WORK_STRUCT_COLOR_SHIFT) &
                ((1 << WORK_STRUCT_COLOR_BITS) - 1);
}

static int work_next_color(int color)
{
        return (color + 1) % WORK_NR_COLORS;
}

static unsigned long pool_offq_flags(struct worker_pool *pool)
{
        return (pool->flags & POOL_BH) ? WORK_OFFQ_BH : 0;
}

/*
 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
 * contain the pointer to the queued pwq.  Once execution starts, the flag
 * is cleared and the high bits contain OFFQ flags and pool ID.
 *
 * set_work_pwq(), set_work_pool_and_clear_pending() and mark_work_canceling()
 * can be used to set the pwq, pool or clear work->data. These functions should
 * only be called while the work is owned - ie. while the PENDING bit is set.
 *
 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
 * corresponding to a work.  Pool is available once the work has been
 * queued anywhere after initialization until it is sync canceled.  pwq is
 * available only while the work item is queued.
 */
static inline void set_work_data(struct work_struct *work, unsigned long data)
{
        WARN_ON_ONCE(!work_pending(work));
        atomic_long_set(&work->data, data | work_static(work));
}

static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
                         unsigned long flags)
{
        set_work_data(work, (unsigned long)pwq | WORK_STRUCT_PENDING |
                      WORK_STRUCT_PWQ | flags);
}

static void set_work_pool_and_keep_pending(struct work_struct *work,
                                           int pool_id, unsigned long flags)
{
        set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) |
                      WORK_STRUCT_PENDING | flags);
}

static void set_work_pool_and_clear_pending(struct work_struct *work,
                                            int pool_id, unsigned long flags)
{
        /*
         * The following wmb is paired with the implied mb in
         * test_and_set_bit(PENDING) and ensures all updates to @work made
         * here are visible to and precede any updates by the next PENDING
         * owner.
         */
        smp_wmb();
        set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) |
                      flags);
        /*
         * The following mb guarantees that previous clear of a PENDING bit
         * will not be reordered with any speculative LOADS or STORES from
         * work->current_func, which is executed afterwards.  This possible
         * reordering can lead to a missed execution on attempt to queue
         * the same @work.  E.g. consider this case:
         *
         *   CPU#0                         CPU#1
         *   ----------------------------  --------------------------------
         *
         * 1  STORE event_indicated
         * 2  queue_work_on() {
         * 3    test_and_set_bit(PENDING)
         * 4 }                             set_..._and_clear_pending() {
         * 5                                 set_work_data() # clear bit
         * 6                                 smp_mb()
         * 7                               work->current_func() {
         * 8                                      LOAD event_indicated
         *                                   }
         *
         * Without an explicit full barrier speculative LOAD on line 8 can
         * be executed before CPU#0 does STORE on line 1.  If that happens,
         * CPU#0 observes the PENDING bit is still set and new execution of
         * a @work is not queued in a hope, that CPU#1 will eventually
         * finish the queued @work.  Meanwhile CPU#1 does not see
         * event_indicated is set, because speculative LOAD was executed
         * before actual STORE.
         */
        smp_mb();
}

static inline struct pool_workqueue *work_struct_pwq(unsigned long data)
{
        return (struct pool_workqueue *)(data & WORK_STRUCT_PWQ_MASK);
}

static struct pool_workqueue *get_work_pwq(struct work_struct *work)
{
        unsigned long data = atomic_long_read(&work->data);

        if (data & WORK_STRUCT_PWQ)
                return work_struct_pwq(data);
        else
                return NULL;
}

/**
 * get_work_pool - return the worker_pool a given work was associated with
 * @work: the work item of interest
 *
 * Pools are created and destroyed under wq_pool_mutex, and allows read
 * access under RCU read lock.  As such, this function should be
 * called under wq_pool_mutex or inside of a rcu_read_lock() region.
 *
 * All fields of the returned pool are accessible as long as the above
 * mentioned locking is in effect.  If the returned pool needs to be used
 * beyond the critical section, the caller is responsible for ensuring the
 * returned pool is and stays online.
 *
 * Return: The worker_pool @work was last associated with.  %NULL if none.
 */
static struct worker_pool *get_work_pool(struct work_struct *work)
{
        unsigned long data = atomic_long_read(&work->data);
        int pool_id;

        assert_rcu_or_pool_mutex();

        if (data & WORK_STRUCT_PWQ)
                return work_struct_pwq(data)->pool;

        pool_id = data >> WORK_OFFQ_POOL_SHIFT;
        if (pool_id == WORK_OFFQ_POOL_NONE)
                return NULL;

        return idr_find(&worker_pool_idr, pool_id);
}

static unsigned long shift_and_mask(unsigned long v, u32 shift, u32 bits)
{
        return (v >> shift) & ((1U << bits) - 1);
}

static void work_offqd_unpack(struct work_offq_data *offqd, unsigned long data)
{
        WARN_ON_ONCE(data & WORK_STRUCT_PWQ);

        offqd->pool_id = shift_and_mask(data, WORK_OFFQ_POOL_SHIFT,
                                        WORK_OFFQ_POOL_BITS);
        offqd->disable = shift_and_mask(data, WORK_OFFQ_DISABLE_SHIFT,
                                        WORK_OFFQ_DISABLE_BITS);
        offqd->flags = data & WORK_OFFQ_FLAG_MASK;
}

static unsigned long work_offqd_pack_flags(struct work_offq_data *offqd)
{
        return ((unsigned long)offqd->disable << WORK_OFFQ_DISABLE_SHIFT) |
                ((unsigned long)offqd->flags);
}

/*
 * Policy functions.  These define the policies on how the global worker
 * pools are managed.  Unless noted otherwise, these functions assume that
 * they're being called with pool->lock held.
 */

/*
 * Need to wake up a worker?  Called from anything but currently
 * running workers.
 *
 * Note that, because unbound workers never contribute to nr_running, this
 * function will always return %true for unbound pools as long as the
 * worklist isn't empty.
 */
static bool need_more_worker(struct worker_pool *pool)
{
        return !list_empty(&pool->worklist) && !pool->nr_running;
}

/* Can I start working?  Called from busy but !running workers. */
static bool may_start_working(struct worker_pool *pool)
{
        return pool->nr_idle;
}

/* Do I need to keep working?  Called from currently running workers. */
static bool keep_working(struct worker_pool *pool)
{
        return !list_empty(&pool->worklist) && (pool->nr_running <= 1);
}

/* Do we need a new worker?  Called from manager. */
static bool need_to_create_worker(struct worker_pool *pool)
{
        return need_more_worker(pool) && !may_start_working(pool);
}

/* Do we have too many workers and should some go away? */
static bool too_many_workers(struct worker_pool *pool)
{
        bool managing = pool->flags & POOL_MANAGER_ACTIVE;
        int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
        int nr_busy = pool->nr_workers - nr_idle;

        return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
}

/**
 * worker_set_flags - set worker flags and adjust nr_running accordingly
 * @worker: self
 * @flags: flags to set
 *
 * Set @flags in @worker->flags and adjust nr_running accordingly.
 */
static inline void worker_set_flags(struct worker *worker, unsigned int flags)
{
        struct worker_pool *pool = worker->pool;

        lockdep_assert_held(&pool->lock);

        /* If transitioning into NOT_RUNNING, adjust nr_running. */
        if ((flags & WORKER_NOT_RUNNING) &&
            !(worker->flags & WORKER_NOT_RUNNING)) {
                pool->nr_running--;
        }

        worker->flags |= flags;
}

/**
 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
 * @worker: self
 * @flags: flags to clear
 *
 * Clear @flags in @worker->flags and adjust nr_running accordingly.
 */
static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
{
        struct worker_pool *pool = worker->pool;
        unsigned int oflags = worker->flags;

        lockdep_assert_held(&pool->lock);

        worker->flags &= ~flags;

        /*
         * If transitioning out of NOT_RUNNING, increment nr_running.  Note
         * that the nested NOT_RUNNING is not a noop.  NOT_RUNNING is mask
         * of multiple flags, not a single flag.
         */
        if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
                if (!(worker->flags & WORKER_NOT_RUNNING))
                        pool->nr_running++;
}

/* Return the first idle worker.  Called with pool->lock held. */
static struct worker *first_idle_worker(struct worker_pool *pool)
{
        if (unlikely(list_empty(&pool->idle_list)))
                return NULL;

        return list_first_entry(&pool->idle_list, struct worker, entry);
}

/**
 * worker_enter_idle - enter idle state
 * @worker: worker which is entering idle state
 *
 * @worker is entering idle state.  Update stats and idle timer if
 * necessary.
 *
 * LOCKING:
 * raw_spin_lock_irq(pool->lock).
 */
static void worker_enter_idle(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;

        if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
            WARN_ON_ONCE(!list_empty(&worker->entry) &&
                         (worker->hentry.next || worker->hentry.pprev)))
                return;

        /* can't use worker_set_flags(), also called from create_worker() */
        worker->flags |= WORKER_IDLE;
        pool->nr_idle++;
        worker->last_active = jiffies;

        /* idle_list is LIFO */
        list_add(&worker->entry, &pool->idle_list);

        if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
                mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);

        /* Sanity check nr_running. */
        WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running);
}

/**
 * worker_leave_idle - leave idle state
 * @worker: worker which is leaving idle state
 *
 * @worker is leaving idle state.  Update stats.
 *
 * LOCKING:
 * raw_spin_lock_irq(pool->lock).
 */
static void worker_leave_idle(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;

        if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
                return;
        worker_clr_flags(worker, WORKER_IDLE);
        pool->nr_idle--;
        list_del_init(&worker->entry);
}

/**
 * find_worker_executing_work - find worker which is executing a work
 * @pool: pool of interest
 * @work: work to find worker for
 *
 * Find a worker which is executing @work on @pool by searching
 * @pool->busy_hash which is keyed by the address of @work.  For a worker
 * to match, its current execution should match the address of @work and
 * its work function.  This is to avoid unwanted dependency between
 * unrelated work executions through a work item being recycled while still
 * being executed.
 *
 * This is a bit tricky.  A work item may be freed once its execution
 * starts and nothing prevents the freed area from being recycled for
 * another work item.  If the same work item address ends up being reused
 * before the original execution finishes, workqueue will identify the
 * recycled work item as currently executing and make it wait until the
 * current execution finishes, introducing an unwanted dependency.
 *
 * This function checks the work item address and work function to avoid
 * false positives.  Note that this isn't complete as one may construct a
 * work function which can introduce dependency onto itself through a
 * recycled work item.  Well, if somebody wants to shoot oneself in the
 * foot that badly, there's only so much we can do, and if such deadlock
 * actually occurs, it should be easy to locate the culprit work function.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 *
 * Return:
 * Pointer to worker which is executing @work if found, %NULL
 * otherwise.
 */
static struct worker *find_worker_executing_work(struct worker_pool *pool,
                                                 struct work_struct *work)
{
        struct worker *worker;

        hash_for_each_possible(pool->busy_hash, worker, hentry,
                               (unsigned long)work)
                if (worker->current_work == work &&
                    worker->current_func == work->func)
                        return worker;

        return NULL;
}

/**
 * move_linked_works - move linked works to a list
 * @work: start of series of works to be scheduled
 * @head: target list to append @work to
 * @nextp: out parameter for nested worklist walking
 *
 * Schedule linked works starting from @work to @head. Work series to be
 * scheduled starts at @work and includes any consecutive work with
 * WORK_STRUCT_LINKED set in its predecessor. See assign_work() for details on
 * @nextp.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 */
static void move_linked_works(struct work_struct *work, struct list_head *head,
                              struct work_struct **nextp)
{
        struct work_struct *n;

        /*
         * Linked worklist will always end before the end of the list,
         * use NULL for list head.
         */
        list_for_each_entry_safe_from(work, n, NULL, entry) {
                list_move_tail(&work->entry, head);
                if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
                        break;
        }

        /*
         * If we're already inside safe list traversal and have moved
         * multiple works to the scheduled queue, the next position
         * needs to be updated.
         */
        if (nextp)
                *nextp = n;
}

/**
 * assign_work - assign a work item and its linked work items to a worker
 * @work: work to assign
 * @worker: worker to assign to
 * @nextp: out parameter for nested worklist walking
 *
 * Assign @work and its linked work items to @worker. If @work is already being
 * executed by another worker in the same pool, it'll be punted there.
 *
 * If @nextp is not NULL, it's updated to point to the next work of the last
 * scheduled work. This allows assign_work() to be nested inside
 * list_for_each_entry_safe().
 *
 * Returns %true if @work was successfully assigned to @worker. %false if @work
 * was punted to another worker already executing it.
 */
static bool assign_work(struct work_struct *work, struct worker *worker,
                        struct work_struct **nextp)
{
        struct worker_pool *pool = worker->pool;
        struct worker *collision;

        lockdep_assert_held(&pool->lock);

        /*
         * A single work shouldn't be executed concurrently by multiple workers.
         * __queue_work() ensures that @work doesn't jump to a different pool
         * while still running in the previous pool. Here, we should ensure that
         * @work is not executed concurrently by multiple workers from the same
         * pool. Check whether anyone is already processing the work. If so,
         * defer the work to the currently executing one.
         */
        collision = find_worker_executing_work(pool, work);
        if (unlikely(collision)) {
                move_linked_works(work, &collision->scheduled, nextp);
                return false;
        }

        move_linked_works(work, &worker->scheduled, nextp);
        return true;
}

static struct irq_work *bh_pool_irq_work(struct worker_pool *pool)
{
        int high = pool->attrs->nice == HIGHPRI_NICE_LEVEL ? 1 : 0;

        return &per_cpu(bh_pool_irq_works, pool->cpu)[high];
}

static void kick_bh_pool(struct worker_pool *pool)
{
#ifdef CONFIG_SMP
        /* see drain_dead_softirq_workfn() for BH_DRAINING */
        if (unlikely(pool->cpu != smp_processor_id() &&
                     !(pool->flags & POOL_BH_DRAINING))) {
                irq_work_queue_on(bh_pool_irq_work(pool), pool->cpu);
                return;
        }
#endif
        if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
                raise_softirq_irqoff(HI_SOFTIRQ);
        else
                raise_softirq_irqoff(TASKLET_SOFTIRQ);
}

/**
 * kick_pool - wake up an idle worker if necessary
 * @pool: pool to kick
 *
 * @pool may have pending work items. Wake up worker if necessary. Returns
 * whether a worker was woken up.
 */
static bool kick_pool(struct worker_pool *pool)
{
        struct worker *worker = first_idle_worker(pool);
        struct task_struct *p;

        lockdep_assert_held(&pool->lock);

        if (!need_more_worker(pool) || !worker)
                return false;

        if (pool->flags & POOL_BH) {
                kick_bh_pool(pool);
                return true;
        }

        p = worker->task;

#ifdef CONFIG_SMP
        /*
         * Idle @worker is about to execute @work and waking up provides an
         * opportunity to migrate @worker at a lower cost by setting the task's
         * wake_cpu field. Let's see if we want to move @worker to improve
         * execution locality.
         *
         * We're waking the worker that went idle the latest and there's some
         * chance that @worker is marked idle but hasn't gone off CPU yet. If
         * so, setting the wake_cpu won't do anything. As this is a best-effort
         * optimization and the race window is narrow, let's leave as-is for
         * now. If this becomes pronounced, we can skip over workers which are
         * still on cpu when picking an idle worker.
         *
         * If @pool has non-strict affinity, @worker might have ended up outside
         * its affinity scope. Repatriate.
         */
        if (!pool->attrs->affn_strict &&
            !cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) {
                struct work_struct *work = list_first_entry(&pool->worklist,
                                                struct work_struct, entry);
                int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask,
                                                          cpu_online_mask);
                if (wake_cpu < nr_cpu_ids) {
                        p->wake_cpu = wake_cpu;
                        get_work_pwq(work)->stats[PWQ_STAT_REPATRIATED]++;
                }
        }
#endif
        wake_up_process(p);
        return true;
}

#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT

/*
 * Concurrency-managed per-cpu work items that hog CPU for longer than
 * wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism,
 * which prevents them from stalling other concurrency-managed work items. If a
 * work function keeps triggering this mechanism, it's likely that the work item
 * should be using an unbound workqueue instead.
 *
 * wq_cpu_intensive_report() tracks work functions which trigger such conditions
 * and report them so that they can be examined and converted to use unbound
 * workqueues as appropriate. To avoid flooding the console, each violating work
 * function is tracked and reported with exponential backoff.
 */
#define WCI_MAX_ENTS 128

struct wci_ent {
        work_func_t                func;
        atomic64_t                cnt;
        struct hlist_node        hash_node;
};

static struct wci_ent wci_ents[WCI_MAX_ENTS];
static int wci_nr_ents;
static DEFINE_RAW_SPINLOCK(wci_lock);
static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS));

static struct wci_ent *wci_find_ent(work_func_t func)
{
        struct wci_ent *ent;

        hash_for_each_possible_rcu(wci_hash, ent, hash_node,
                                   (unsigned long)func) {
                if (ent->func == func)
                        return ent;
        }
        return NULL;
}

static void wq_cpu_intensive_report(work_func_t func)
{
        struct wci_ent *ent;

restart:
        ent = wci_find_ent(func);
        if (ent) {
                u64 cnt;

                /*
                 * Start reporting from the warning_thresh and back off
                 * exponentially.
                 */
                cnt = atomic64_inc_return_relaxed(&ent->cnt);
                if (wq_cpu_intensive_warning_thresh &&
                    cnt >= wq_cpu_intensive_warning_thresh &&
                    is_power_of_2(cnt + 1 - wq_cpu_intensive_warning_thresh))
                        printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n",
                                        ent->func, wq_cpu_intensive_thresh_us,
                                        atomic64_read(&ent->cnt));
                return;
        }

        /*
         * @func is a new violation. Allocate a new entry for it. If wcn_ents[]
         * is exhausted, something went really wrong and we probably made enough
         * noise already.
         */
        if (wci_nr_ents >= WCI_MAX_ENTS)
                return;

        raw_spin_lock(&wci_lock);

        if (wci_nr_ents >= WCI_MAX_ENTS) {
                raw_spin_unlock(&wci_lock);
                return;
        }

        if (wci_find_ent(func)) {
                raw_spin_unlock(&wci_lock);
                goto restart;
        }

        ent = &wci_ents[wci_nr_ents++];
        ent->func = func;
        atomic64_set(&ent->cnt, 0);
        hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func);

        raw_spin_unlock(&wci_lock);

        goto restart;
}

#else        /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
static void wq_cpu_intensive_report(work_func_t func) {}
#endif        /* CONFIG_WQ_CPU_INTENSIVE_REPORT */

/**
 * wq_worker_running - a worker is running again
 * @task: task waking up
 *
 * This function is called when a worker returns from schedule()
 */
void wq_worker_running(struct task_struct *task)
{
        struct worker *worker = kthread_data(task);

        if (!READ_ONCE(worker->sleeping))
                return;

        /*
         * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
         * and the nr_running increment below, we may ruin the nr_running reset
         * and leave with an unexpected pool->nr_running == 1 on the newly unbound
         * pool. Protect against such race.
         */
        preempt_disable();
        if (!(worker->flags & WORKER_NOT_RUNNING))
                worker->pool->nr_running++;
        preempt_enable();

        /*
         * CPU intensive auto-detection cares about how long a work item hogged
         * CPU without sleeping. Reset the starting timestamp on wakeup.
         */
        worker->current_at = worker->task->se.sum_exec_runtime;

        WRITE_ONCE(worker->sleeping, 0);
}

/**
 * wq_worker_sleeping - a worker is going to sleep
 * @task: task going to sleep
 *
 * This function is called from schedule() when a busy worker is
 * going to sleep.
 */
void wq_worker_sleeping(struct task_struct *task)
{
        struct worker *worker = kthread_data(task);
        struct worker_pool *pool;

        /*
         * Rescuers, which may not have all the fields set up like normal
         * workers, also reach here, let's not access anything before
         * checking NOT_RUNNING.
         */
        if (worker->flags & WORKER_NOT_RUNNING)
                return;

        pool = worker->pool;

        /* Return if preempted before wq_worker_running() was reached */
        if (READ_ONCE(worker->sleeping))
                return;

        WRITE_ONCE(worker->sleeping, 1);
        raw_spin_lock_irq(&pool->lock);

        /*
         * Recheck in case unbind_workers() preempted us. We don't
         * want to decrement nr_running after the worker is unbound
         * and nr_running has been reset.
         */
        if (worker->flags & WORKER_NOT_RUNNING) {
                raw_spin_unlock_irq(&pool->lock);
                return;
        }

        pool->nr_running--;
        if (kick_pool(pool))
                worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++;

        raw_spin_unlock_irq(&pool->lock);
}

/**
 * wq_worker_tick - a scheduler tick occurred while a kworker is running
 * @task: task currently running
 *
 * Called from sched_tick(). We're in the IRQ context and the current
 * worker's fields which follow the 'K' locking rule can be accessed safely.
 */
void wq_worker_tick(struct task_struct *task)
{
        struct worker *worker = kthread_data(task);
        struct pool_workqueue *pwq = worker->current_pwq;
        struct worker_pool *pool = worker->pool;

        if (!pwq)
                return;

        pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;

        if (!wq_cpu_intensive_thresh_us)
                return;

        /*
         * If the current worker is concurrency managed and hogged the CPU for
         * longer than wq_cpu_intensive_thresh_us, it's automatically marked
         * CPU_INTENSIVE to avoid stalling other concurrency-managed work items.
         *
         * Set @worker->sleeping means that @worker is in the process of
         * switching out voluntarily and won't be contributing to
         * @pool->nr_running until it wakes up. As wq_worker_sleeping() also
         * decrements ->nr_running, setting CPU_INTENSIVE here can lead to
         * double decrements. The task is releasing the CPU anyway. Let's skip.
         * We probably want to make this prettier in the future.
         */
        if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||
            worker->task->se.sum_exec_runtime - worker->current_at <
            wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
                return;

        raw_spin_lock(&pool->lock);

        worker_set_flags(worker, WORKER_CPU_INTENSIVE);
        wq_cpu_intensive_report(worker->current_func);
        pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;

        if (kick_pool(pool))
                pwq->stats[PWQ_STAT_CM_WAKEUP]++;

        raw_spin_unlock(&pool->lock);
}

/**
 * wq_worker_last_func - retrieve worker's last work function
 * @task: Task to retrieve last work function of.
 *
 * Determine the last function a worker executed. This is called from
 * the scheduler to get a worker's last known identity.
 *
 * CONTEXT:
 * raw_spin_lock_irq(rq->lock)
 *
 * This function is called during schedule() when a kworker is going
 * to sleep. It's used by psi to identify aggregation workers during
 * dequeuing, to allow periodic aggregation to shut-off when that
 * worker is the last task in the system or cgroup to go to sleep.
 *
 * As this function doesn't involve any workqueue-related locking, it
 * only returns stable values when called from inside the scheduler's
 * queuing and dequeuing paths, when @task, which must be a kworker,
 * is guaranteed to not be processing any works.
 *
 * Return:
 * The last work function %current executed as a worker, NULL if it
 * hasn't executed any work yet.
 */
work_func_t wq_worker_last_func(struct task_struct *task)
{
        struct worker *worker = kthread_data(task);

        return worker->last_func;
}

/**
 * wq_node_nr_active - Determine wq_node_nr_active to use
 * @wq: workqueue of interest
 * @node: NUMA node, can be %NUMA_NO_NODE
 *
 * Determine wq_node_nr_active to use for @wq on @node. Returns:
 *
 * - %NULL for per-cpu workqueues as they don't need to use shared nr_active.
 *
 * - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE.
 *
 * - Otherwise, node_nr_active[@node].
 */
static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq,
                                                   int node)
{
        if (!(wq->flags & WQ_UNBOUND))
                return NULL;

        if (node == NUMA_NO_NODE)
                node = nr_node_ids;

        return wq->node_nr_active[node];
}

/**
 * wq_update_node_max_active - Update per-node max_actives to use
 * @wq: workqueue to update
 * @off_cpu: CPU that's going down, -1 if a CPU is not going down
 *
 * Update @wq->node_nr_active[]->max. @wq must be unbound. max_active is
 * distributed among nodes according to the proportions of numbers of online
 * cpus. The result is always between @wq->min_active and max_active.
 */
static void wq_update_node_max_active(struct workqueue_struct *wq, int off_cpu)
{
        struct cpumask *effective = unbound_effective_cpumask(wq);
        int min_active = READ_ONCE(wq->min_active);
        int max_active = READ_ONCE(wq->max_active);
        int total_cpus, node;

        lockdep_assert_held(&wq->mutex);

        if (!wq_topo_initialized)
                return;

        if (off_cpu >= 0 && !cpumask_test_cpu(off_cpu, effective))
                off_cpu = -1;

        total_cpus = cpumask_weight_and(effective, cpu_online_mask);
        if (off_cpu >= 0)
                total_cpus--;

        /* If all CPUs of the wq get offline, use the default values */
        if (unlikely(!total_cpus)) {
                for_each_node(node)
                        wq_node_nr_active(wq, node)->max = min_active;

                wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
                return;
        }

        for_each_node(node) {
                int node_cpus;

                node_cpus = cpumask_weight_and(effective, cpumask_of_node(node));
                if (off_cpu >= 0 && cpu_to_node(off_cpu) == node)
                        node_cpus--;

                wq_node_nr_active(wq, node)->max =
                        clamp(DIV_ROUND_UP(max_active * node_cpus, total_cpus),
                              min_active, max_active);
        }

        wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
}

/**
 * get_pwq - get an extra reference on the specified pool_workqueue
 * @pwq: pool_workqueue to get
 *
 * Obtain an extra reference on @pwq.  The caller should guarantee that
 * @pwq has positive refcnt and be holding the matching pool->lock.
 */
static void get_pwq(struct pool_workqueue *pwq)
{
        lockdep_assert_held(&pwq->pool->lock);
        WARN_ON_ONCE(pwq->refcnt <= 0);
        pwq->refcnt++;
}

/**
 * put_pwq - put a pool_workqueue reference
 * @pwq: pool_workqueue to put
 *
 * Drop a reference of @pwq.  If its refcnt reaches zero, schedule its
 * destruction.  The caller should be holding the matching pool->lock.
 */
static void put_pwq(struct pool_workqueue *pwq)
{
        lockdep_assert_held(&pwq->pool->lock);
        if (likely(--pwq->refcnt))
                return;
        /*
         * @pwq can't be released under pool->lock, bounce to a dedicated
         * kthread_worker to avoid A-A deadlocks.
         */
        kthread_queue_work(pwq_release_worker, &pwq->release_work);
}

/**
 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
 * @pwq: pool_workqueue to put (can be %NULL)
 *
 * put_pwq() with locking.  This function also allows %NULL @pwq.
 */
static void put_pwq_unlocked(struct pool_workqueue *pwq)
{
        if (pwq) {
                /*
                 * As both pwqs and pools are RCU protected, the
                 * following lock operations are safe.
                 */
                raw_spin_lock_irq(&pwq->pool->lock);
                put_pwq(pwq);
                raw_spin_unlock_irq(&pwq->pool->lock);
        }
}

static bool pwq_is_empty(struct pool_workqueue *pwq)
{
        return !pwq->nr_active && list_empty(&pwq->inactive_works);
}

static void __pwq_activate_work(struct pool_workqueue *pwq,
                                struct work_struct *work)
{
        unsigned long *wdb = work_data_bits(work);

        WARN_ON_ONCE(!(*wdb & WORK_STRUCT_INACTIVE));
        trace_workqueue_activate_work(work);
        if (list_empty(&pwq->pool->worklist))
                pwq->pool->watchdog_ts = jiffies;
        move_linked_works(work, &pwq->pool->worklist, NULL);
        __clear_bit(WORK_STRUCT_INACTIVE_BIT, wdb);
}

static bool tryinc_node_nr_active(struct wq_node_nr_active *nna)
{
        int max = READ_ONCE(nna->max);

        while (true) {
                int old, tmp;

                old = atomic_read(&nna->nr);
                if (old >= max)
                        return false;
                tmp = atomic_cmpxchg_relaxed(&nna->nr, old, old + 1);
                if (tmp == old)
                        return true;
        }
}

/**
 * pwq_tryinc_nr_active - Try to increment nr_active for a pwq
 * @pwq: pool_workqueue of interest
 * @fill: max_active may have increased, try to increase concurrency level
 *
 * Try to increment nr_active for @pwq. Returns %true if an nr_active count is
 * successfully obtained. %false otherwise.
 */
static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
{
        struct workqueue_struct *wq = pwq->wq;
        struct worker_pool *pool = pwq->pool;
        struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);
        bool obtained = false;

        lockdep_assert_held(&pool->lock);

        if (!nna) {
                /* BH or per-cpu workqueue, pwq->nr_active is sufficient */
                obtained = pwq->nr_active < READ_ONCE(wq->max_active);
                goto out;
        }

        if (unlikely(pwq->plugged))
                return false;

        /*
         * Unbound workqueue uses per-node shared nr_active $nna. If @pwq is
         * already waiting on $nna, pwq_dec_nr_active() will maintain the
         * concurrency level. Don't jump the line.
         *
         * We need to ignore the pending test after max_active has increased as
         * pwq_dec_nr_active() can only maintain the concurrency level but not
         * increase it. This is indicated by @fill.
         */
        if (!list_empty(&pwq->pending_node) && likely(!fill))
                goto out;

        obtained = tryinc_node_nr_active(nna);
        if (obtained)
                goto out;

        /*
         * Lockless acquisition failed. Lock, add ourself to $nna->pending_pwqs
         * and try again. The smp_mb() is paired with the implied memory barrier
         * of atomic_dec_return() in pwq_dec_nr_active() to ensure that either
         * we see the decremented $nna->nr or they see non-empty
         * $nna->pending_pwqs.
         */
        raw_spin_lock(&nna->lock);

        if (list_empty(&pwq->pending_node))
                list_add_tail(&pwq->pending_node, &nna->pending_pwqs);
        else if (likely(!fill))
                goto out_unlock;

        smp_mb();

        obtained = tryinc_node_nr_active(nna);

        /*
         * If @fill, @pwq might have already been pending. Being spuriously
         * pending in cold paths doesn't affect anything. Let's leave it be.
         */
        if (obtained && likely(!fill))
                list_del_init(&pwq->pending_node);

out_unlock:
        raw_spin_unlock(&nna->lock);
out:
        if (obtained)
                pwq->nr_active++;
        return obtained;
}

/**
 * pwq_activate_first_inactive - Activate the first inactive work item on a pwq
 * @pwq: pool_workqueue of interest
 * @fill: max_active may have increased, try to increase concurrency level
 *
 * Activate the first inactive work item of @pwq if available and allowed by
 * max_active limit.
 *
 * Returns %true if an inactive work item has been activated. %false if no
 * inactive work item is found or max_active limit is reached.
 */
static bool pwq_activate_first_inactive(struct pool_workqueue *pwq, bool fill)
{
        struct work_struct *work =
                list_first_entry_or_null(&pwq->inactive_works,
                                         struct work_struct, entry);

        if (work && pwq_tryinc_nr_active(pwq, fill)) {
                __pwq_activate_work(pwq, work);
                return true;
        } else {
                return false;
        }
}

/**
 * unplug_oldest_pwq - unplug the oldest pool_workqueue
 * @wq: workqueue_struct where its oldest pwq is to be unplugged
 *
 * This function should only be called for ordered workqueues where only the
 * oldest pwq is unplugged, the others are plugged to suspend execution to
 * ensure proper work item ordering::
 *
 *    dfl_pwq --------------+     [P] - plugged
 *                          |
 *                          v
 *    pwqs -> A -> B [P] -> C [P] (newest)
 *            |    |        |
 *            1    3        5
 *            |    |        |
 *            2    4        6
 *
 * When the oldest pwq is drained and removed, this function should be called
 * to unplug the next oldest one to start its work item execution. Note that
 * pwq's are linked into wq->pwqs with the oldest first, so the first one in
 * the list is the oldest.
 */
static void unplug_oldest_pwq(struct workqueue_struct *wq)
{
        struct pool_workqueue *pwq;

        lockdep_assert_held(&wq->mutex);

        /* Caller should make sure that pwqs isn't empty before calling */
        pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue,
                                       pwqs_node);
        raw_spin_lock_irq(&pwq->pool->lock);
        if (pwq->plugged) {
                pwq->plugged = false;
                if (pwq_activate_first_inactive(pwq, true))
                        kick_pool(pwq->pool);
        }
        raw_spin_unlock_irq(&pwq->pool->lock);
}

/**
 * node_activate_pending_pwq - Activate a pending pwq on a wq_node_nr_active
 * @nna: wq_node_nr_active to activate a pending pwq for
 * @caller_pool: worker_pool the caller is locking
 *
 * Activate a pwq in @nna->pending_pwqs. Called with @caller_pool locked.
 * @caller_pool may be unlocked and relocked to lock other worker_pools.
 */
static void node_activate_pending_pwq(struct wq_node_nr_active *nna,
                                      struct worker_pool *caller_pool)
{
        struct worker_pool *locked_pool = caller_pool;
        struct pool_workqueue *pwq;
        struct work_struct *work;

        lockdep_assert_held(&caller_pool->lock);

        raw_spin_lock(&nna->lock);
retry:
        pwq = list_first_entry_or_null(&nna->pending_pwqs,
                                       struct pool_workqueue, pending_node);
        if (!pwq)
                goto out_unlock;

        /*
         * If @pwq is for a different pool than @locked_pool, we need to lock
         * @pwq->pool->lock. Let's trylock first. If unsuccessful, do the unlock
         * / lock dance. For that, we also need to release @nna->lock as it's
         * nested inside pool locks.
         */
        if (pwq->pool != locked_pool) {
                raw_spin_unlock(&locked_pool->lock);
                locked_pool = pwq->pool;
                if (!raw_spin_trylock(&locked_pool->lock)) {
                        raw_spin_unlock(&nna->lock);
                        raw_spin_lock(&locked_pool->lock);
                        raw_spin_lock(&nna->lock);
                        goto retry;
                }
        }

        /*
         * $pwq may not have any inactive work items due to e.g. cancellations.
         * Drop it from pending_pwqs and see if there's another one.
         */
        work = list_first_entry_or_null(&pwq->inactive_works,
                                        struct work_struct, entry);
        if (!work) {
                list_del_init(&pwq->pending_node);
                goto retry;
        }

        /*
         * Acquire an nr_active count and activate the inactive work item. If
         * $pwq still has inactive work items, rotate it to the end of the
         * pending_pwqs so that we round-robin through them. This means that
         * inactive work items are not activated in queueing order which is fine
         * given that there has never been any ordering across different pwqs.
         */
        if (likely(tryinc_node_nr_active(nna))) {
                pwq->nr_active++;
                __pwq_activate_work(pwq, work);

                if (list_empty(&pwq->inactive_works))
                        list_del_init(&pwq->pending_node);
                else
                        list_move_tail(&pwq->pending_node, &nna->pending_pwqs);

                /* if activating a foreign pool, make sure it's running */
                if (pwq->pool != caller_pool)
                        kick_pool(pwq->pool);
        }

out_unlock:
        raw_spin_unlock(&nna->lock);
        if (locked_pool != caller_pool) {
                raw_spin_unlock(&locked_pool->lock);
                raw_spin_lock(&caller_pool->lock);
        }
}

/**
 * pwq_dec_nr_active - Retire an active count
 * @pwq: pool_workqueue of interest
 *
 * Decrement @pwq's nr_active and try to activate the first inactive work item.
 * For unbound workqueues, this function may temporarily drop @pwq->pool->lock.
 */
static void pwq_dec_nr_active(struct pool_workqueue *pwq)
{
        struct worker_pool *pool = pwq->pool;
        struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);

        lockdep_assert_held(&pool->lock);

        /*
         * @pwq->nr_active should be decremented for both percpu and unbound
         * workqueues.
         */
        pwq->nr_active--;

        /*
         * For a percpu workqueue, it's simple. Just need to kick the first
         * inactive work item on @pwq itself.
         */
        if (!nna) {
                pwq_activate_first_inactive(pwq, false);
                return;
        }

        /*
         * If @pwq is for an unbound workqueue, it's more complicated because
         * multiple pwqs and pools may be sharing the nr_active count. When a
         * pwq needs to wait for an nr_active count, it puts itself on
         * $nna->pending_pwqs. The following atomic_dec_return()'s implied
         * memory barrier is paired with smp_mb() in pwq_tryinc_nr_active() to
         * guarantee that either we see non-empty pending_pwqs or they see
         * decremented $nna->nr.
         *
         * $nna->max may change as CPUs come online/offline and @pwq->wq's
         * max_active gets updated. However, it is guaranteed to be equal to or
         * larger than @pwq->wq->min_active which is above zero unless freezing.
         * This maintains the forward progress guarantee.
         */
        if (atomic_dec_return(&nna->nr) >= READ_ONCE(nna->max))
                return;

        if (!list_empty(&nna->pending_pwqs))
                node_activate_pending_pwq(nna, pool);
}

/**
 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
 * @pwq: pwq of interest
 * @work_data: work_data of work which left the queue
 *
 * A work either has completed or is removed from pending queue,
 * decrement nr_in_flight of its pwq and handle workqueue flushing.
 *
 * NOTE:
 * For unbound workqueues, this function may temporarily drop @pwq->pool->lock
 * and thus should be called after all other state updates for the in-flight
 * work item is complete.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 */
static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
{
        int color = get_work_color(work_data);

        if (!(work_data & WORK_STRUCT_INACTIVE))
                pwq_dec_nr_active(pwq);

        pwq->nr_in_flight[color]--;

        /* is flush in progress and are we at the flushing tip? */
        if (likely(pwq->flush_color != color))
                goto out_put;

        /* are there still in-flight works? */
        if (pwq->nr_in_flight[color])
                goto out_put;

        /* this pwq is done, clear flush_color */
        pwq->flush_color = -1;

        /*
         * If this was the last pwq, wake up the first flusher.  It
         * will handle the rest.
         */
        if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
                complete(&pwq->wq->first_flusher->done);
out_put:
        put_pwq(pwq);
}

/**
 * try_to_grab_pending - steal work item from worklist and disable irq
 * @work: work item to steal
 * @cflags: %WORK_CANCEL_ flags
 * @irq_flags: place to store irq state
 *
 * Try to grab PENDING bit of @work.  This function can handle @work in any
 * stable state - idle, on timer or on worklist.
 *
 * Return:
 *
 *  ========        ================================================================
 *  1                if @work was pending and we successfully stole PENDING
 *  0                if @work was idle and we claimed PENDING
 *  -EAGAIN        if PENDING couldn't be grabbed at the moment, safe to busy-retry
 *  ========        ================================================================
 *
 * Note:
 * On >= 0 return, the caller owns @work's PENDING bit.  To avoid getting
 * interrupted while holding PENDING and @work off queue, irq must be
 * disabled on entry.  This, combined with delayed_work->timer being
 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
 *
 * On successful return, >= 0, irq is disabled and the caller is
 * responsible for releasing it using local_irq_restore(*@irq_flags).
 *
 * This function is safe to call from any context including IRQ handler.
 */
static int try_to_grab_pending(struct work_struct *work, u32 cflags,
                               unsigned long *irq_flags)
{
        struct worker_pool *pool;
        struct pool_workqueue *pwq;

        local_irq_save(*irq_flags);

        /* try to steal the timer if it exists */
        if (cflags & WORK_CANCEL_DELAYED) {
                struct delayed_work *dwork = to_delayed_work(work);

                /*
                 * dwork->timer is irqsafe.  If del_timer() fails, it's
                 * guaranteed that the timer is not queued anywhere and not
                 * running on the local CPU.
                 */
                if (likely(del_timer(&dwork->timer)))
                        return 1;
        }

        /* try to claim PENDING the normal way */
        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
                return 0;

        rcu_read_lock();
        /*
         * The queueing is in progress, or it is already queued. Try to
         * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
         */
        pool = get_work_pool(work);
        if (!pool)
                goto fail;

        raw_spin_lock(&pool->lock);
        /*
         * work->data is guaranteed to point to pwq only while the work
         * item is queued on pwq->wq, and both updating work->data to point
         * to pwq on queueing and to pool on dequeueing are done under
         * pwq->pool->lock.  This in turn guarantees that, if work->data
         * points to pwq which is associated with a locked pool, the work
         * item is currently queued on that pool.
         */
        pwq = get_work_pwq(work);
        if (pwq && pwq->pool == pool) {
                unsigned long work_data = *work_data_bits(work);

                debug_work_deactivate(work);

                /*
                 * A cancelable inactive work item must be in the
                 * pwq->inactive_works since a queued barrier can't be
                 * canceled (see the comments in insert_wq_barrier()).
                 *
                 * An inactive work item cannot be deleted directly because
                 * it might have linked barrier work items which, if left
                 * on the inactive_works list, will confuse pwq->nr_active
                 * management later on and cause stall.  Move the linked
                 * barrier work items to the worklist when deleting the grabbed
                 * item. Also keep WORK_STRUCT_INACTIVE in work_data, so that
                 * it doesn't participate in nr_active management in later
                 * pwq_dec_nr_in_flight().
                 */
                if (work_data & WORK_STRUCT_INACTIVE)
                        move_linked_works(work, &pwq->pool->worklist, NULL);

                list_del_init(&work->entry);

                /*
                 * work->data points to pwq iff queued. Let's point to pool. As
                 * this destroys work->data needed by the next step, stash it.
                 */
                set_work_pool_and_keep_pending(work, pool->id,
                                               pool_offq_flags(pool));

                /* must be the last step, see the function comment */
                pwq_dec_nr_in_flight(pwq, work_data);

                raw_spin_unlock(&pool->lock);
                rcu_read_unlock();
                return 1;
        }
        raw_spin_unlock(&pool->lock);
fail:
        rcu_read_unlock();
        local_irq_restore(*irq_flags);
        return -EAGAIN;
}

/**
 * work_grab_pending - steal work item from worklist and disable irq
 * @work: work item to steal
 * @cflags: %WORK_CANCEL_ flags
 * @irq_flags: place to store IRQ state
 *
 * Grab PENDING bit of @work. @work can be in any stable state - idle, on timer
 * or on worklist.
 *
 * Can be called from any context. IRQ is disabled on return with IRQ state
 * stored in *@irq_flags. The caller is responsible for re-enabling it using
 * local_irq_restore().
 *
 * Returns %true if @work was pending. %false if idle.
 */
static bool work_grab_pending(struct work_struct *work, u32 cflags,
                              unsigned long *irq_flags)
{
        int ret;

        while (true) {
                ret = try_to_grab_pending(work, cflags, irq_flags);
                if (ret >= 0)
                        return ret;
                cpu_relax();
        }
}

/**
 * insert_work - insert a work into a pool
 * @pwq: pwq @work belongs to
 * @work: work to insert
 * @head: insertion point
 * @extra_flags: extra WORK_STRUCT_* flags to set
 *
 * Insert @work which belongs to @pwq after @head.  @extra_flags is or'd to
 * work_struct flags.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 */
static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
                        struct list_head *head, unsigned int extra_flags)
{
        debug_work_activate(work);

        /* record the work call stack in order to print it in KASAN reports */
        kasan_record_aux_stack_noalloc(work);

        /* we own @work, set data and link */
        set_work_pwq(work, pwq, extra_flags);
        list_add_tail(&work->entry, head);
        get_pwq(pwq);
}

/*
 * Test whether @work is being queued from another work executing on the
 * same workqueue.
 */
static bool is_chained_work(struct workqueue_struct *wq)
{
        struct worker *worker;

        worker = current_wq_worker();
        /*
         * Return %true iff I'm a worker executing a work item on @wq.  If
         * I'm @worker, it's safe to dereference it without locking.
         */
        return worker && worker->current_pwq->wq == wq;
}

/*
 * When queueing an unbound work item to a wq, prefer local CPU if allowed
 * by wq_unbound_cpumask.  Otherwise, round robin among the allowed ones to
 * avoid perturbing sensitive tasks.
 */
static int wq_select_unbound_cpu(int cpu)
{
        int new_cpu;

        if (likely(!wq_debug_force_rr_cpu)) {
                if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
                        return cpu;
        } else {
                pr_warn_once("workqueue: round-robin CPU selection forced, expect performance impact\n");
        }

        new_cpu = __this_cpu_read(wq_rr_cpu_last);
        new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
        if (unlikely(new_cpu >= nr_cpu_ids)) {
                new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
                if (unlikely(new_cpu >= nr_cpu_ids))
                        return cpu;
        }
        __this_cpu_write(wq_rr_cpu_last, new_cpu);

        return new_cpu;
}

static void __queue_work(int cpu, struct workqueue_struct *wq,
                         struct work_struct *work)
{
        struct pool_workqueue *pwq;
        struct worker_pool *last_pool, *pool;
        unsigned int work_flags;
        unsigned int req_cpu = cpu;

        /*
         * While a work item is PENDING && off queue, a task trying to
         * steal the PENDING will busy-loop waiting for it to either get
         * queued or lose PENDING.  Grabbing PENDING and queueing should
         * happen with IRQ disabled.
         */
        lockdep_assert_irqs_disabled();

        /*
         * For a draining wq, only works from the same workqueue are
         * allowed. The __WQ_DESTROYING helps to spot the issue that
         * queues a new work item to a wq after destroy_workqueue(wq).
         */
        if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) &&
                     WARN_ON_ONCE(!is_chained_work(wq))))
                return;
        rcu_read_lock();
retry:
        /* pwq which will be used unless @work is executing elsewhere */
        if (req_cpu == WORK_CPU_UNBOUND) {
                if (wq->flags & WQ_UNBOUND)
                        cpu = wq_select_unbound_cpu(raw_smp_processor_id());
                else
                        cpu = raw_smp_processor_id();
        }

        pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu));
        pool = pwq->pool;

        /*
         * If @work was previously on a different pool, it might still be
         * running there, in which case the work needs to be queued on that
         * pool to guarantee non-reentrancy.
         *
         * For ordered workqueue, work items must be queued on the newest pwq
         * for accurate order management.  Guaranteed order also guarantees
         * non-reentrancy.  See the comments above unplug_oldest_pwq().
         */
        last_pool = get_work_pool(work);
        if (last_pool && last_pool != pool && !(wq->flags & __WQ_ORDERED)) {
                struct worker *worker;

                raw_spin_lock(&last_pool->lock);

                worker = find_worker_executing_work(last_pool, work);

                if (worker && worker->current_pwq->wq == wq) {
                        pwq = worker->current_pwq;
                        pool = pwq->pool;
                        WARN_ON_ONCE(pool != last_pool);
                } else {
                        /* meh... not running there, queue here */
                        raw_spin_unlock(&last_pool->lock);
                        raw_spin_lock(&pool->lock);
                }
        } else {
                raw_spin_lock(&pool->lock);
        }

        /*
         * pwq is determined and locked. For unbound pools, we could have raced
         * with pwq release and it could already be dead. If its refcnt is zero,
         * repeat pwq selection. Note that unbound pwqs never die without
         * another pwq replacing it in cpu_pwq or while work items are executing
         * on it, so the retrying is guaranteed to make forward-progress.
         */
        if (unlikely(!pwq->refcnt)) {
                if (wq->flags & WQ_UNBOUND) {
                        raw_spin_unlock(&pool->lock);
                        cpu_relax();
                        goto retry;
                }
                /* oops */
                WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
                          wq->name, cpu);
        }

        /* pwq determined, queue */
        trace_workqueue_queue_work(req_cpu, pwq, work);

        if (WARN_ON(!list_empty(&work->entry)))
                goto out;

        pwq->nr_in_flight[pwq->work_color]++;
        work_flags = work_color_to_flags(pwq->work_color);

        /*
         * Limit the number of concurrently active work items to max_active.
         * @work must also queue behind existing inactive work items to maintain
         * ordering when max_active changes. See wq_adjust_max_active().
         */
        if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq, false)) {
                if (list_empty(&pool->worklist))
                        pool->watchdog_ts = jiffies;

                trace_workqueue_activate_work(work);
                insert_work(pwq, work, &pool->worklist, work_flags);
                kick_pool(pool);
        } else {
                work_flags |= WORK_STRUCT_INACTIVE;
                insert_work(pwq, work, &pwq->inactive_works, work_flags);
        }

out:
        raw_spin_unlock(&pool->lock);
        rcu_read_unlock();
}

static bool clear_pending_if_disabled(struct work_struct *work)
{
        unsigned long data = *work_data_bits(work);
        struct work_offq_data offqd;

        if (likely((data & WORK_STRUCT_PWQ) ||
                   !(data & WORK_OFFQ_DISABLE_MASK)))
                return false;

        work_offqd_unpack(&offqd, data);
        set_work_pool_and_clear_pending(work, offqd.pool_id,
                                        work_offqd_pack_flags(&offqd));
        return true;
}

/**
 * queue_work_on - queue work on specific cpu
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
 * @work: work to queue
 *
 * We queue the work to a specific CPU, the caller must ensure it
 * can't go away.  Callers that fail to ensure that the specified
 * CPU cannot go away will execute on a randomly chosen CPU.
 * But note well that callers specifying a CPU that never has been
 * online will get a splat.
 *
 * Return: %false if @work was already on a queue, %true otherwise.
 */
bool queue_work_on(int cpu, struct workqueue_struct *wq,
                   struct work_struct *work)
{
        bool ret = false;
        unsigned long irq_flags;

        local_irq_save(irq_flags);

        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
            !clear_pending_if_disabled(work)) {
                __queue_work(cpu, wq, work);
                ret = true;
        }

        local_irq_restore(irq_flags);
        return ret;
}
EXPORT_SYMBOL(queue_work_on);

/**
 * select_numa_node_cpu - Select a CPU based on NUMA node
 * @node: NUMA node ID that we want to select a CPU from
 *
 * This function will attempt to find a "random" cpu available on a given
 * node. If there are no CPUs available on the given node it will return
 * WORK_CPU_UNBOUND indicating that we should just schedule to any
 * available CPU if we need to schedule this work.
 */
static int select_numa_node_cpu(int node)
{
        int cpu;

        /* Delay binding to CPU if node is not valid or online */
        if (node < 0 || node >= MAX_NUMNODES || !node_online(node))
                return WORK_CPU_UNBOUND;

        /* Use local node/cpu if we are already there */
        cpu = raw_smp_processor_id();
        if (node == cpu_to_node(cpu))
                return cpu;

        /* Use "random" otherwise know as "first" online CPU of node */
        cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);

        /* If CPU is valid return that, otherwise just defer */
        return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
}

/**
 * queue_work_node - queue work on a "random" cpu for a given NUMA node
 * @node: NUMA node that we are targeting the work for
 * @wq: workqueue to use
 * @work: work to queue
 *
 * We queue the work to a "random" CPU within a given NUMA node. The basic
 * idea here is to provide a way to somehow associate work with a given
 * NUMA node.
 *
 * This function will only make a best effort attempt at getting this onto
 * the right NUMA node. If no node is requested or the requested node is
 * offline then we just fall back to standard queue_work behavior.
 *
 * Currently the "random" CPU ends up being the first available CPU in the
 * intersection of cpu_online_mask and the cpumask of the node, unless we
 * are running on the node. In that case we just use the current CPU.
 *
 * Return: %false if @work was already on a queue, %true otherwise.
 */
bool queue_work_node(int node, struct workqueue_struct *wq,
                     struct work_struct *work)
{
        unsigned long irq_flags;
        bool ret = false;

        /*
         * This current implementation is specific to unbound workqueues.
         * Specifically we only return the first available CPU for a given
         * node instead of cycling through individual CPUs within the node.
         *
         * If this is used with a per-cpu workqueue then the logic in
         * workqueue_select_cpu_near would need to be updated to allow for
         * some round robin type logic.
         */
        WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));

        local_irq_save(irq_flags);

        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
            !clear_pending_if_disabled(work)) {
                int cpu = select_numa_node_cpu(node);

                __queue_work(cpu, wq, work);
                ret = true;
        }

        local_irq_restore(irq_flags);
        return ret;
}
EXPORT_SYMBOL_GPL(queue_work_node);

void delayed_work_timer_fn(struct timer_list *t)
{
        struct delayed_work *dwork = from_timer(dwork, t, timer);

        /* should have been called from irqsafe timer with irq already off */
        __queue_work(dwork->cpu, dwork->wq, &dwork->work);
}
EXPORT_SYMBOL(delayed_work_timer_fn);

static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
                                struct delayed_work *dwork, unsigned long delay)
{
        struct timer_list *timer = &dwork->timer;
        struct work_struct *work = &dwork->work;

        WARN_ON_ONCE(!wq);
        WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
        WARN_ON_ONCE(timer_pending(timer));
        WARN_ON_ONCE(!list_empty(&work->entry));

        /*
         * If @delay is 0, queue @dwork->work immediately.  This is for
         * both optimization and correctness.  The earliest @timer can
         * expire is on the closest next tick and delayed_work users depend
         * on that there's no such delay when @delay is 0.
         */
        if (!delay) {
                __queue_work(cpu, wq, &dwork->work);
                return;
        }

        dwork->wq = wq;
        dwork->cpu = cpu;
        timer->expires = jiffies + delay;

        if (housekeeping_enabled(HK_TYPE_TIMER)) {
                /* If the current cpu is a housekeeping cpu, use it. */
                cpu = smp_processor_id();
                if (!housekeeping_test_cpu(cpu, HK_TYPE_TIMER))
                        cpu = housekeeping_any_cpu(HK_TYPE_TIMER);
                add_timer_on(timer, cpu);
        } else {
                if (likely(cpu == WORK_CPU_UNBOUND))
                        add_timer_global(timer);
                else
                        add_timer_on(timer, cpu);
        }
}

/**
 * queue_delayed_work_on - queue work on specific CPU after delay
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
 * @dwork: work to queue
 * @delay: number of jiffies to wait before queueing
 *
 * Return: %false if @work was already on a queue, %true otherwise.  If
 * @delay is zero and @dwork is idle, it will be scheduled for immediate
 * execution.
 */
bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
                           struct delayed_work *dwork, unsigned long delay)
{
        struct work_struct *work = &dwork->work;
        bool ret = false;
        unsigned long irq_flags;

        /* read the comment in __queue_work() */
        local_irq_save(irq_flags);

        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
            !clear_pending_if_disabled(work)) {
                __queue_delayed_work(cpu, wq, dwork, delay);
                ret = true;
        }

        local_irq_restore(irq_flags);
        return ret;
}
EXPORT_SYMBOL(queue_delayed_work_on);

/**
 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
 * @dwork: work to queue
 * @delay: number of jiffies to wait before queueing
 *
 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
 * modify @dwork's timer so that it expires after @delay.  If @delay is
 * zero, @work is guaranteed to be scheduled immediately regardless of its
 * current state.
 *
 * Return: %false if @dwork was idle and queued, %true if @dwork was
 * pending and its timer was modified.
 *
 * This function is safe to call from any context including IRQ handler.
 * See try_to_grab_pending() for details.
 */
bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
                         struct delayed_work *dwork, unsigned long delay)
{
        unsigned long irq_flags;
        bool ret;

        ret = work_grab_pending(&dwork->work, WORK_CANCEL_DELAYED, &irq_flags);

        if (!clear_pending_if_disabled(&dwork->work))
                __queue_delayed_work(cpu, wq, dwork, delay);

        local_irq_restore(irq_flags);
        return ret;
}
EXPORT_SYMBOL_GPL(mod_delayed_work_on);

static void rcu_work_rcufn(struct rcu_head *rcu)
{
        struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);

        /* read the comment in __queue_work() */
        local_irq_disable();
        __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
        local_irq_enable();
}

/**
 * queue_rcu_work - queue work after a RCU grace period
 * @wq: workqueue to use
 * @rwork: work to queue
 *
 * Return: %false if @rwork was already pending, %true otherwise.  Note
 * that a full RCU grace period is guaranteed only after a %true return.
 * While @rwork is guaranteed to be executed after a %false return, the
 * execution may happen before a full RCU grace period has passed.
 */
bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
{
        struct work_struct *work = &rwork->work;

        /*
         * rcu_work can't be canceled or disabled. Warn if the user reached
         * inside @rwork and disabled the inner work.
         */
        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
            !WARN_ON_ONCE(clear_pending_if_disabled(work))) {
                rwork->wq = wq;
                call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
                return true;
        }

        return false;
}
EXPORT_SYMBOL(queue_rcu_work);

static struct worker *alloc_worker(int node)
{
        struct worker *worker;

        worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
        if (worker) {
                INIT_LIST_HEAD(&worker->entry);
                INIT_LIST_HEAD(&worker->scheduled);
                INIT_LIST_HEAD(&worker->node);
                /* on creation a worker is in !idle && prep state */
                worker->flags = WORKER_PREP;
        }
        return worker;
}

static cpumask_t *pool_allowed_cpus(struct worker_pool *pool)
{
        if (pool->cpu < 0 && pool->attrs->affn_strict)
                return pool->attrs->__pod_cpumask;
        else
                return pool->attrs->cpumask;
}

/**
 * worker_attach_to_pool() - attach a worker to a pool
 * @worker: worker to be attached
 * @pool: the target pool
 *
 * Attach @worker to @pool.  Once attached, the %WORKER_UNBOUND flag and
 * cpu-binding of @worker are kept coordinated with the pool across
 * cpu-[un]hotplugs.
 */
static void worker_attach_to_pool(struct worker *worker,
                                  struct worker_pool *pool)
{
        mutex_lock(&wq_pool_attach_mutex);

        /*
         * The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains stable
         * across this function. See the comments above the flag definition for
         * details. BH workers are, while per-CPU, always DISASSOCIATED.
         */
        if (pool->flags & POOL_DISASSOCIATED) {
                worker->flags |= WORKER_UNBOUND;
        } else {
                WARN_ON_ONCE(pool->flags & POOL_BH);
                kthread_set_per_cpu(worker->task, pool->cpu);
        }

        if (worker->rescue_wq)
                set_cpus_allowed_ptr(worker->task, pool_allowed_cpus(pool));

        list_add_tail(&worker->node, &pool->workers);
        worker->pool = pool;

        mutex_unlock(&wq_pool_attach_mutex);
}

static void unbind_worker(struct worker *worker)
{
        lockdep_assert_held(&wq_pool_attach_mutex);

        kthread_set_per_cpu(worker->task, -1);
        if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))
                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0);
        else
                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);
}


static void detach_worker(struct worker *worker)
{
        lockdep_assert_held(&wq_pool_attach_mutex);

        unbind_worker(worker);
        list_del(&worker->node);
}

/**
 * worker_detach_from_pool() - detach a worker from its pool
 * @worker: worker which is attached to its pool
 *
 * Undo the attaching which had been done in worker_attach_to_pool().  The
 * caller worker shouldn't access to the pool after detached except it has
 * other reference to the pool.
 */
static void worker_detach_from_pool(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;

        /* there is one permanent BH worker per CPU which should never detach */
        WARN_ON_ONCE(pool->flags & POOL_BH);

        mutex_lock(&wq_pool_attach_mutex);
        detach_worker(worker);
        worker->pool = NULL;
        mutex_unlock(&wq_pool_attach_mutex);

        /* clear leftover flags without pool->lock after it is detached */
        worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
}

static int format_worker_id(char *buf, size_t size, struct worker *worker,
                            struct worker_pool *pool)
{
        if (worker->rescue_wq)
                return scnprintf(buf, size, "kworker/R-%s",
                                 worker->rescue_wq->name);

        if (pool) {
                if (pool->cpu >= 0)
                        return scnprintf(buf, size, "kworker/%d:%d%s",
                                         pool->cpu, worker->id,
                                         pool->attrs->nice < 0  ? "H" : "");
                else
                        return scnprintf(buf, size, "kworker/u%d:%d",
                                         pool->id, worker->id);
        } else {
                return scnprintf(buf, size, "kworker/dying");
        }
}

/**
 * create_worker - create a new workqueue worker
 * @pool: pool the new worker will belong to
 *
 * Create and start a new worker which is attached to @pool.
 *
 * CONTEXT:
 * Might sleep.  Does GFP_KERNEL allocations.
 *
 * Return:
 * Pointer to the newly created worker.
 */
static struct worker *create_worker(struct worker_pool *pool)
{
        struct worker *worker;
        int id;

        /* ID is needed to determine kthread name */
        id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
        if (id < 0) {
                pr_err_once("workqueue: Failed to allocate a worker ID: %pe\n",
                            ERR_PTR(id));
                return NULL;
        }

        worker = alloc_worker(pool->node);
        if (!worker) {
                pr_err_once("workqueue: Failed to allocate a worker\n");
                goto fail;
        }

        worker->id = id;

        if (!(pool->flags & POOL_BH)) {
                char id_buf[WORKER_ID_LEN];

                format_worker_id(id_buf, sizeof(id_buf), worker, pool);
                worker->task = kthread_create_on_node(worker_thread, worker,
                                                      pool->node, "%s", id_buf);
                if (IS_ERR(worker->task)) {
                        if (PTR_ERR(worker->task) == -EINTR) {
                                pr_err("workqueue: Interrupted when creating a worker thread \"%s\"\n",
                                       id_buf);
                        } else {
                                pr_err_once("workqueue: Failed to create a worker thread: %pe",
                                            worker->task);
                        }
                        goto fail;
                }

                set_user_nice(worker->task, pool->attrs->nice);
                kthread_bind_mask(worker->task, pool_allowed_cpus(pool));
        }

        /* successful, attach the worker to the pool */
        worker_attach_to_pool(worker, pool);

        /* start the newly created worker */
        raw_spin_lock_irq(&pool->lock);

        worker->pool->nr_workers++;
        worker_enter_idle(worker);

        /*
         * @worker is waiting on a completion in kthread() and will trigger hung
         * check if not woken up soon. As kick_pool() is noop if @pool is empty,
         * wake it up explicitly.
         */
        if (worker->task)
                wake_up_process(worker->task);

        raw_spin_unlock_irq(&pool->lock);

        return worker;

fail:
        ida_free(&pool->worker_ida, id);
        kfree(worker);
        return NULL;
}

static void detach_dying_workers(struct list_head *cull_list)
{
        struct worker *worker;

        list_for_each_entry(worker, cull_list, entry)
                detach_worker(worker);
}

static void reap_dying_workers(struct list_head *cull_list)
{
        struct worker *worker, *tmp;

        list_for_each_entry_safe(worker, tmp, cull_list, entry) {
                list_del_init(&worker->entry);
                kthread_stop_put(worker->task);
                kfree(worker);
        }
}

/**
 * set_worker_dying - Tag a worker for destruction
 * @worker: worker to be destroyed
 * @list: transfer worker away from its pool->idle_list and into list
 *
 * Tag @worker for destruction and adjust @pool stats accordingly.  The worker
 * should be idle.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 */
static void set_worker_dying(struct worker *worker, struct list_head *list)
{
        struct worker_pool *pool = worker->pool;

        lockdep_assert_held(&pool->lock);
        lockdep_assert_held(&wq_pool_attach_mutex);

        /* sanity check frenzy */
        if (WARN_ON(worker->current_work) ||
            WARN_ON(!list_empty(&worker->scheduled)) ||
            WARN_ON(!(worker->flags & WORKER_IDLE)))
                return;

        pool->nr_workers--;
        pool->nr_idle--;

        worker->flags |= WORKER_DIE;

        list_move(&worker->entry, list);

        /* get an extra task struct reference for later kthread_stop_put() */
        get_task_struct(worker->task);
}

/**
 * idle_worker_timeout - check if some idle workers can now be deleted.
 * @t: The pool's idle_timer that just expired
 *
 * The timer is armed in worker_enter_idle(). Note that it isn't disarmed in
 * worker_leave_idle(), as a worker flicking between idle and active while its
 * pool is at the too_many_workers() tipping point would cause too much timer
 * housekeeping overhead. Since IDLE_WORKER_TIMEOUT is long enough, we just let
 * it expire and re-evaluate things from there.
 */
static void idle_worker_timeout(struct timer_list *t)
{
        struct worker_pool *pool = from_timer(pool, t, idle_timer);
        bool do_cull = false;

        if (work_pending(&pool->idle_cull_work))
                return;

        raw_spin_lock_irq(&pool->lock);

        if (too_many_workers(pool)) {
                struct worker *worker;
                unsigned long expires;

                /* idle_list is kept in LIFO order, check the last one */
                worker = list_last_entry(&pool->idle_list, struct worker, entry);
                expires = worker->last_active + IDLE_WORKER_TIMEOUT;
                do_cull = !time_before(jiffies, expires);

                if (!do_cull)
                        mod_timer(&pool->idle_timer, expires);
        }
        raw_spin_unlock_irq(&pool->lock);

        if (do_cull)
                queue_work(system_unbound_wq, &pool->idle_cull_work);
}

/**
 * idle_cull_fn - cull workers that have been idle for too long.
 * @work: the pool's work for handling these idle workers
 *
 * This goes through a pool's idle workers and gets rid of those that have been
 * idle for at least IDLE_WORKER_TIMEOUT seconds.
 *
 * We don't want to disturb isolated CPUs because of a pcpu kworker being
 * culled, so this also resets worker affinity. This requires a sleepable
 * context, hence the split between timer callback and work item.
 */
static void idle_cull_fn(struct work_struct *work)
{
        struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work);
        LIST_HEAD(cull_list);

        /*
         * Grabbing wq_pool_attach_mutex here ensures an already-running worker
         * cannot proceed beyong set_pf_worker() in its self-destruct path.
         * This is required as a previously-preempted worker could run after
         * set_worker_dying() has happened but before detach_dying_workers() did.
         */
        mutex_lock(&wq_pool_attach_mutex);
        raw_spin_lock_irq(&pool->lock);

        while (too_many_workers(pool)) {
                struct worker *worker;
                unsigned long expires;

                worker = list_last_entry(&pool->idle_list, struct worker, entry);
                expires = worker->last_active + IDLE_WORKER_TIMEOUT;

                if (time_before(jiffies, expires)) {
                        mod_timer(&pool->idle_timer, expires);
                        break;
                }

                set_worker_dying(worker, &cull_list);
        }

        raw_spin_unlock_irq(&pool->lock);
        detach_dying_workers(&cull_list);
        mutex_unlock(&wq_pool_attach_mutex);

        reap_dying_workers(&cull_list);
}

static void send_mayday(struct work_struct *work)
{
        struct pool_workqueue *pwq = get_work_pwq(work);
        struct workqueue_struct *wq = pwq->wq;

        lockdep_assert_held(&wq_mayday_lock);

        if (!wq->rescuer)
                return;

        /* mayday mayday mayday */
        if (list_empty(&pwq->mayday_node)) {
                /*
                 * If @pwq is for an unbound wq, its base ref may be put at
                 * any time due to an attribute change.  Pin @pwq until the
                 * rescuer is done with it.
                 */
                get_pwq(pwq);
                list_add_tail(&pwq->mayday_node, &wq->maydays);
                wake_up_process(wq->rescuer->task);
                pwq->stats[PWQ_STAT_MAYDAY]++;
        }
}

static void pool_mayday_timeout(struct timer_list *t)
{
        struct worker_pool *pool = from_timer(pool, t, mayday_timer);
        struct work_struct *work;

        raw_spin_lock_irq(&pool->lock);
        raw_spin_lock(&wq_mayday_lock);                /* for wq->maydays */

        if (need_to_create_worker(pool)) {
                /*
                 * We've been trying to create a new worker but
                 * haven't been successful.  We might be hitting an
                 * allocation deadlock.  Send distress signals to
                 * rescuers.
                 */
                list_for_each_entry(work, &pool->worklist, entry)
                        send_mayday(work);
        }

        raw_spin_unlock(&wq_mayday_lock);
        raw_spin_unlock_irq(&pool->lock);

        mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
}

/**
 * maybe_create_worker - create a new worker if necessary
 * @pool: pool to create a new worker for
 *
 * Create a new worker for @pool if necessary.  @pool is guaranteed to
 * have at least one idle worker on return from this function.  If
 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
 * sent to all rescuers with works scheduled on @pool to resolve
 * possible allocation deadlock.
 *
 * On return, need_to_create_worker() is guaranteed to be %false and
 * may_start_working() %true.
 *
 * LOCKING:
 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
 * multiple times.  Does GFP_KERNEL allocations.  Called only from
 * manager.
 */
static void maybe_create_worker(struct worker_pool *pool)
__releases(&pool->lock)
__acquires(&pool->lock)
{
restart:
        raw_spin_unlock_irq(&pool->lock);

        /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
        mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);

        while (true) {
                if (create_worker(pool) || !need_to_create_worker(pool))
                        break;

                schedule_timeout_interruptible(CREATE_COOLDOWN);

                if (!need_to_create_worker(pool))
                        break;
        }

        del_timer_sync(&pool->mayday_timer);
        raw_spin_lock_irq(&pool->lock);
        /*
         * This is necessary even after a new worker was just successfully
         * created as @pool->lock was dropped and the new worker might have
         * already become busy.
         */
        if (need_to_create_worker(pool))
                goto restart;
}

/**
 * manage_workers - manage worker pool
 * @worker: self
 *
 * Assume the manager role and manage the worker pool @worker belongs
 * to.  At any given time, there can be only zero or one manager per
 * pool.  The exclusion is handled automatically by this function.
 *
 * The caller can safely start processing works on false return.  On
 * true return, it's guaranteed that need_to_create_worker() is false
 * and may_start_working() is true.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
 * multiple times.  Does GFP_KERNEL allocations.
 *
 * Return:
 * %false if the pool doesn't need management and the caller can safely
 * start processing works, %true if management function was performed and
 * the conditions that the caller verified before calling the function may
 * no longer be true.
 */
static bool manage_workers(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;

        if (pool->flags & POOL_MANAGER_ACTIVE)
                return false;

        pool->flags |= POOL_MANAGER_ACTIVE;
        pool->manager = worker;

        maybe_create_worker(pool);

        pool->manager = NULL;
        pool->flags &= ~POOL_MANAGER_ACTIVE;
        rcuwait_wake_up(&manager_wait);
        return true;
}

/**
 * process_one_work - process single work
 * @worker: self
 * @work: work to process
 *
 * Process @work.  This function contains all the logics necessary to
 * process a single work including synchronization against and
 * interaction with other workers on the same cpu, queueing and
 * flushing.  As long as context requirement is met, any worker can
 * call this function to process a work.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
 */
static void process_one_work(struct worker *worker, struct work_struct *work)
__releases(&pool->lock)
__acquires(&pool->lock)
{
        struct pool_workqueue *pwq = get_work_pwq(work);
        struct worker_pool *pool = worker->pool;
        unsigned long work_data;
        int lockdep_start_depth, rcu_start_depth;
        bool bh_draining = pool->flags & POOL_BH_DRAINING;
#ifdef CONFIG_LOCKDEP
        /*
         * It is permissible to free the struct work_struct from
         * inside the function that is called from it, this we need to
         * take into account for lockdep too.  To avoid bogus "held
         * lock freed" warnings as well as problems when looking into
         * work->lockdep_map, make a copy and use that here.
         */
        struct lockdep_map lockdep_map;

        lockdep_copy_map(&lockdep_map, &work->lockdep_map);
#endif
        /* ensure we're on the correct CPU */
        WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
                     raw_smp_processor_id() != pool->cpu);

        /* claim and dequeue */
        debug_work_deactivate(work);
        hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
        worker->current_work = work;
        worker->current_func = work->func;
        worker->current_pwq = pwq;
        if (worker->task)
                worker->current_at = worker->task->se.sum_exec_runtime;
        work_data = *work_data_bits(work);
        worker->current_color = get_work_color(work_data);

        /*
         * Record wq name for cmdline and debug reporting, may get
         * overridden through set_worker_desc().
         */
        strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);

        list_del_init(&work->entry);

        /*
         * CPU intensive works don't participate in concurrency management.
         * They're the scheduler's responsibility.  This takes @worker out
         * of concurrency management and the next code block will chain
         * execution of the pending work items.
         */
        if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
                worker_set_flags(worker, WORKER_CPU_INTENSIVE);

        /*
         * Kick @pool if necessary. It's always noop for per-cpu worker pools
         * since nr_running would always be >= 1 at this point. This is used to
         * chain execution of the pending work items for WORKER_NOT_RUNNING
         * workers such as the UNBOUND and CPU_INTENSIVE ones.
         */
        kick_pool(pool);

        /*
         * Record the last pool and clear PENDING which should be the last
         * update to @work.  Also, do this inside @pool->lock so that
         * PENDING and queued state changes happen together while IRQ is
         * disabled.
         */
        set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool));

        pwq->stats[PWQ_STAT_STARTED]++;
        raw_spin_unlock_irq(&pool->lock);

        rcu_start_depth = rcu_preempt_depth();
        lockdep_start_depth = lockdep_depth(current);
        /* see drain_dead_softirq_workfn() */
        if (!bh_draining)
                lock_map_acquire(pwq->wq->lockdep_map);
        lock_map_acquire(&lockdep_map);
        /*
         * Strictly speaking we should mark the invariant state without holding
         * any locks, that is, before these two lock_map_acquire()'s.
         *
         * However, that would result in:
         *
         *   A(W1)
         *   WFC(C)
         *                A(W1)
         *                C(C)
         *
         * Which would create W1->C->W1 dependencies, even though there is no
         * actual deadlock possible. There are two solutions, using a
         * read-recursive acquire on the work(queue) 'locks', but this will then
         * hit the lockdep limitation on recursive locks, or simply discard
         * these locks.
         *
         * AFAICT there is no possible deadlock scenario between the
         * flush_work() and complete() primitives (except for single-threaded
         * workqueues), so hiding them isn't a problem.
         */
        lockdep_invariant_state(true);
        trace_workqueue_execute_start(work);
        worker->current_func(work);
        /*
         * While we must be careful to not use "work" after this, the trace
         * point will only record its address.
         */
        trace_workqueue_execute_end(work, worker->current_func);
        pwq->stats[PWQ_STAT_COMPLETED]++;
        lock_map_release(&lockdep_map);
        if (!bh_draining)
                lock_map_release(pwq->wq->lockdep_map);

        if (unlikely((worker->task && in_atomic()) ||
                     lockdep_depth(current) != lockdep_start_depth ||
                     rcu_preempt_depth() != rcu_start_depth)) {
                pr_err("BUG: workqueue leaked atomic, lock or RCU: %s[%d]\n"
                       "     preempt=0x%08x lock=%d->%d RCU=%d->%d workfn=%ps\n",
                       current->comm, task_pid_nr(current), preempt_count(),
                       lockdep_start_depth, lockdep_depth(current),
                       rcu_start_depth, rcu_preempt_depth(),
                       worker->current_func);
                debug_show_held_locks(current);
                dump_stack();
        }

        /*
         * The following prevents a kworker from hogging CPU on !PREEMPTION
         * kernels, where a requeueing work item waiting for something to
         * happen could deadlock with stop_machine as such work item could
         * indefinitely requeue itself while all other CPUs are trapped in
         * stop_machine. At the same time, report a quiescent RCU state so
         * the same condition doesn't freeze RCU.
         */
        if (worker->task)
                cond_resched();

        raw_spin_lock_irq(&pool->lock);

        /*
         * In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked
         * CPU intensive by wq_worker_tick() if @work hogged CPU longer than
         * wq_cpu_intensive_thresh_us. Clear it.
         */
        worker_clr_flags(worker, WORKER_CPU_INTENSIVE);

        /* tag the worker for identification in schedule() */
        worker->last_func = worker->current_func;

        /* we're done with it, release */
        hash_del(&worker->hentry);
        worker->current_work = NULL;
        worker->current_func = NULL;
        worker->current_pwq = NULL;
        worker->current_color = INT_MAX;

        /* must be the last step, see the function comment */
        pwq_dec_nr_in_flight(pwq, work_data);
}

/**
 * process_scheduled_works - process scheduled works
 * @worker: self
 *
 * Process all scheduled works.  Please note that the scheduled list
 * may change while processing a work, so this function repeatedly
 * fetches a work from the top and executes it.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
 * multiple times.
 */
static void process_scheduled_works(struct worker *worker)
{
        struct work_struct *work;
        bool first = true;

        while ((work = list_first_entry_or_null(&worker->scheduled,
                                                struct work_struct, entry))) {
                if (first) {
                        worker->pool->watchdog_ts = jiffies;
                        first = false;
                }
                process_one_work(worker, work);
        }
}

static void set_pf_worker(bool val)
{
        mutex_lock(&wq_pool_attach_mutex);
        if (val)
                current->flags |= PF_WQ_WORKER;
        else
                current->flags &= ~PF_WQ_WORKER;
        mutex_unlock(&wq_pool_attach_mutex);
}

/**
 * worker_thread - the worker thread function
 * @__worker: self
 *
 * The worker thread function.  All workers belong to a worker_pool -
 * either a per-cpu one or dynamic unbound one.  These workers process all
 * work items regardless of their specific target workqueue.  The only
 * exception is work items which belong to workqueues with a rescuer which
 * will be explained in rescuer_thread().
 *
 * Return: 0
 */
static int worker_thread(void *__worker)
{
        struct worker *worker = __worker;
        struct worker_pool *pool = worker->pool;

        /* tell the scheduler that this is a workqueue worker */
        set_pf_worker(true);
woke_up:
        raw_spin_lock_irq(&pool->lock);

        /* am I supposed to die? */
        if (unlikely(worker->flags & WORKER_DIE)) {
                raw_spin_unlock_irq(&pool->lock);
                set_pf_worker(false);
                /*
                 * The worker is dead and PF_WQ_WORKER is cleared, worker->pool
                 * shouldn't be accessed, reset it to NULL in case otherwise.
                 */
                worker->pool = NULL;
                ida_free(&pool->worker_ida, worker->id);
                return 0;
        }

        worker_leave_idle(worker);
recheck:
        /* no more worker necessary? */
        if (!need_more_worker(pool))
                goto sleep;

        /* do we need to manage? */
        if (unlikely(!may_start_working(pool)) && manage_workers(worker))
                goto recheck;

        /*
         * ->scheduled list can only be filled while a worker is
         * preparing to process a work or actually processing it.
         * Make sure nobody diddled with it while I was sleeping.
         */
        WARN_ON_ONCE(!list_empty(&worker->scheduled));

        /*
         * Finish PREP stage.  We're guaranteed to have at least one idle
         * worker or that someone else has already assumed the manager
         * role.  This is where @worker starts participating in concurrency
         * management if applicable and concurrency management is restored
         * after being rebound.  See rebind_workers() for details.
         */
        worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);

        do {
                struct work_struct *work =
                        list_first_entry(&pool->worklist,
                                         struct work_struct, entry);

                if (assign_work(work, worker, NULL))
                        process_scheduled_works(worker);
        } while (keep_working(pool));

        worker_set_flags(worker, WORKER_PREP);
sleep:
        /*
         * pool->lock is held and there's no work to process and no need to
         * manage, sleep.  Workers are woken up only while holding
         * pool->lock or from local cpu, so setting the current state
         * before releasing pool->lock is enough to prevent losing any
         * event.
         */
        worker_enter_idle(worker);
        __set_current_state(TASK_IDLE);
        raw_spin_unlock_irq(&pool->lock);
        schedule();
        goto woke_up;
}

/**
 * rescuer_thread - the rescuer thread function
 * @__rescuer: self
 *
 * Workqueue rescuer thread function.  There's one rescuer for each
 * workqueue which has WQ_MEM_RECLAIM set.
 *
 * Regular work processing on a pool may block trying to create a new
 * worker which uses GFP_KERNEL allocation which has slight chance of
 * developing into deadlock if some works currently on the same queue
 * need to be processed to satisfy the GFP_KERNEL allocation.  This is
 * the problem rescuer solves.
 *
 * When such condition is possible, the pool summons rescuers of all
 * workqueues which have works queued on the pool and let them process
 * those works so that forward progress can be guaranteed.
 *
 * This should happen rarely.
 *
 * Return: 0
 */
static int rescuer_thread(void *__rescuer)
{
        struct worker *rescuer = __rescuer;
        struct workqueue_struct *wq = rescuer->rescue_wq;
        bool should_stop;

        set_user_nice(current, RESCUER_NICE_LEVEL);

        /*
         * Mark rescuer as worker too.  As WORKER_PREP is never cleared, it
         * doesn't participate in concurrency management.
         */
        set_pf_worker(true);
repeat:
        set_current_state(TASK_IDLE);

        /*
         * By the time the rescuer is requested to stop, the workqueue
         * shouldn't have any work pending, but @wq->maydays may still have
         * pwq(s) queued.  This can happen by non-rescuer workers consuming
         * all the work items before the rescuer got to them.  Go through
         * @wq->maydays processing before acting on should_stop so that the
         * list is always empty on exit.
         */
        should_stop = kthread_should_stop();

        /* see whether any pwq is asking for help */
        raw_spin_lock_irq(&wq_mayday_lock);

        while (!list_empty(&wq->maydays)) {
                struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
                                        struct pool_workqueue, mayday_node);
                struct worker_pool *pool = pwq->pool;
                struct work_struct *work, *n;

                __set_current_state(TASK_RUNNING);
                list_del_init(&pwq->mayday_node);

                raw_spin_unlock_irq(&wq_mayday_lock);

                worker_attach_to_pool(rescuer, pool);

                raw_spin_lock_irq(&pool->lock);

                /*
                 * Slurp in all works issued via this workqueue and
                 * process'em.
                 */
                WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
                list_for_each_entry_safe(work, n, &pool->worklist, entry) {
                        if (get_work_pwq(work) == pwq &&
                            assign_work(work, rescuer, &n))
                                pwq->stats[PWQ_STAT_RESCUED]++;
                }

                if (!list_empty(&rescuer->scheduled)) {
                        process_scheduled_works(rescuer);

                        /*
                         * The above execution of rescued work items could
                         * have created more to rescue through
                         * pwq_activate_first_inactive() or chained
                         * queueing.  Let's put @pwq back on mayday list so
                         * that such back-to-back work items, which may be
                         * being used to relieve memory pressure, don't
                         * incur MAYDAY_INTERVAL delay inbetween.
                         */
                        if (pwq->nr_active && need_to_create_worker(pool)) {
                                raw_spin_lock(&wq_mayday_lock);
                                /*
                                 * Queue iff we aren't racing destruction
                                 * and somebody else hasn't queued it already.
                                 */
                                if (wq->rescuer && list_empty(&pwq->mayday_node)) {
                                        get_pwq(pwq);
                                        list_add_tail(&pwq->mayday_node, &wq->maydays);
                                }
                                raw_spin_unlock(&wq_mayday_lock);
                        }
                }

                /*
                 * Put the reference grabbed by send_mayday().  @pool won't
                 * go away while we're still attached to it.
                 */
                put_pwq(pwq);

                /*
                 * Leave this pool. Notify regular workers; otherwise, we end up
                 * with 0 concurrency and stalling the execution.
                 */
                kick_pool(pool);

                raw_spin_unlock_irq(&pool->lock);

                worker_detach_from_pool(rescuer);

                raw_spin_lock_irq(&wq_mayday_lock);
        }

        raw_spin_unlock_irq(&wq_mayday_lock);

        if (should_stop) {
                __set_current_state(TASK_RUNNING);
                set_pf_worker(false);
                return 0;
        }

        /* rescuers should never participate in concurrency management */
        WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
        schedule();
        goto repeat;
}

static void bh_worker(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;
        int nr_restarts = BH_WORKER_RESTARTS;
        unsigned long end = jiffies + BH_WORKER_JIFFIES;

        raw_spin_lock_irq(&pool->lock);
        worker_leave_idle(worker);

        /*
         * This function follows the structure of worker_thread(). See there for
         * explanations on each step.
         */
        if (!need_more_worker(pool))
                goto done;

        WARN_ON_ONCE(!list_empty(&worker->scheduled));
        worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);

        do {
                struct work_struct *work =
                        list_first_entry(&pool->worklist,
                                         struct work_struct, entry);

                if (assign_work(work, worker, NULL))
                        process_scheduled_works(worker);
        } while (keep_working(pool) &&
                 --nr_restarts && time_before(jiffies, end));

        worker_set_flags(worker, WORKER_PREP);
done:
        worker_enter_idle(worker);
        kick_pool(pool);
        raw_spin_unlock_irq(&pool->lock);
}

/*
 * TODO: Convert all tasklet users to workqueue and use softirq directly.
 *
 * This is currently called from tasklet[_hi]action() and thus is also called
 * whenever there are tasklets to run. Let's do an early exit if there's nothing
 * queued. Once conversion from tasklet is complete, the need_more_worker() test
 * can be dropped.
 *
 * After full conversion, we'll add worker->softirq_action, directly use the
 * softirq action and obtain the worker pointer from the softirq_action pointer.
 */
void workqueue_softirq_action(bool highpri)
{
        struct worker_pool *pool =
                &per_cpu(bh_worker_pools, smp_processor_id())[highpri];
        if (need_more_worker(pool))
                bh_worker(list_first_entry(&pool->workers, struct worker, node));
}

struct wq_drain_dead_softirq_work {
        struct work_struct        work;
        struct worker_pool        *pool;
        struct completion        done;
};

static void drain_dead_softirq_workfn(struct work_struct *work)
{
        struct wq_drain_dead_softirq_work *dead_work =
                container_of(work, struct wq_drain_dead_softirq_work, work);
        struct worker_pool *pool = dead_work->pool;
        bool repeat;

        /*
         * @pool's CPU is dead and we want to execute its still pending work
         * items from this BH work item which is running on a different CPU. As
         * its CPU is dead, @pool can't be kicked and, as work execution path
         * will be nested, a lockdep annotation needs to be suppressed. Mark
         * @pool with %POOL_BH_DRAINING for the special treatments.
         */
        raw_spin_lock_irq(&pool->lock);
        pool->flags |= POOL_BH_DRAINING;
        raw_spin_unlock_irq(&pool->lock);

        bh_worker(list_first_entry(&pool->workers, struct worker, node));

        raw_spin_lock_irq(&pool->lock);
        pool->flags &= ~POOL_BH_DRAINING;
        repeat = need_more_worker(pool);
        raw_spin_unlock_irq(&pool->lock);

        /*
         * bh_worker() might hit consecutive execution limit and bail. If there
         * still are pending work items, reschedule self and return so that we
         * don't hog this CPU's BH.
         */
        if (repeat) {
                if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
                        queue_work(system_bh_highpri_wq, work);
                else
                        queue_work(system_bh_wq, work);
        } else {
                complete(&dead_work->done);
        }
}

/*
 * @cpu is dead. Drain the remaining BH work items on the current CPU. It's
 * possible to allocate dead_work per CPU and avoid flushing. However, then we
 * have to worry about draining overlapping with CPU coming back online or
 * nesting (one CPU's dead_work queued on another CPU which is also dead and so
 * on). Let's keep it simple and drain them synchronously. These are BH work
 * items which shouldn't be requeued on the same pool. Shouldn't take long.
 */
void workqueue_softirq_dead(unsigned int cpu)
{
        int i;

        for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
                struct worker_pool *pool = &per_cpu(bh_worker_pools, cpu)[i];
                struct wq_drain_dead_softirq_work dead_work;

                if (!need_more_worker(pool))
                        continue;

                INIT_WORK_ONSTACK(&dead_work.work, drain_dead_softirq_workfn);
                dead_work.pool = pool;
                init_completion(&dead_work.done);

                if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
                        queue_work(system_bh_highpri_wq, &dead_work.work);
                else
                        queue_work(system_bh_wq, &dead_work.work);

                wait_for_completion(&dead_work.done);
                destroy_work_on_stack(&dead_work.work);
        }
}

/**
 * check_flush_dependency - check for flush dependency sanity
 * @target_wq: workqueue being flushed
 * @target_work: work item being flushed (NULL for workqueue flushes)
 *
 * %current is trying to flush the whole @target_wq or @target_work on it.
 * If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
 * reclaiming memory or running on a workqueue which doesn't have
 * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
 * a deadlock.
 */
static void check_flush_dependency(struct workqueue_struct *target_wq,
                                   struct work_struct *target_work)
{
        work_func_t target_func = target_work ? target_work->func : NULL;
        struct worker *worker;

        if (target_wq->flags & WQ_MEM_RECLAIM)
                return;

        worker = current_wq_worker();

        WARN_ONCE(current->flags & PF_MEMALLOC,
                  "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
                  current->pid, current->comm, target_wq->name, target_func);
        WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
                              (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),
                  "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
                  worker->current_pwq->wq->name, worker->current_func,
                  target_wq->name, target_func);
}

struct wq_barrier {
        struct work_struct        work;
        struct completion        done;
        struct task_struct        *task;        /* purely informational */
};

static void wq_barrier_func(struct work_struct *work)
{
        struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
        complete(&barr->done);
}

/**
 * insert_wq_barrier - insert a barrier work
 * @pwq: pwq to insert barrier into
 * @barr: wq_barrier to insert
 * @target: target work to attach @barr to
 * @worker: worker currently executing @target, NULL if @target is not executing
 *
 * @barr is linked to @target such that @barr is completed only after
 * @target finishes execution.  Please note that the ordering
 * guarantee is observed only with respect to @target and on the local
 * cpu.
 *
 * Currently, a queued barrier can't be canceled.  This is because
 * try_to_grab_pending() can't determine whether the work to be
 * grabbed is at the head of the queue and thus can't clear LINKED
 * flag of the previous work while there must be a valid next work
 * after a work with LINKED flag set.
 *
 * Note that when @worker is non-NULL, @target may be modified
 * underneath us, so we can't reliably determine pwq from @target.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 */
static void insert_wq_barrier(struct pool_workqueue *pwq,
                              struct wq_barrier *barr,
                              struct work_struct *target, struct worker *worker)
{
        static __maybe_unused struct lock_class_key bh_key, thr_key;
        unsigned int work_flags = 0;
        unsigned int work_color;
        struct list_head *head;

        /*
         * debugobject calls are safe here even with pool->lock locked
         * as we know for sure that this will not trigger any of the
         * checks and call back into the fixup functions where we
         * might deadlock.
         *
         * BH and threaded workqueues need separate lockdep keys to avoid
         * spuriously triggering "inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W}
         * usage".
         */
        INIT_WORK_ONSTACK_KEY(&barr->work, wq_barrier_func,
                              (pwq->wq->flags & WQ_BH) ? &bh_key : &thr_key);
        __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));

        init_completion_map(&barr->done, &target->lockdep_map);

        barr->task = current;

        /* The barrier work item does not participate in nr_active. */
        work_flags |= WORK_STRUCT_INACTIVE;

        /*
         * If @target is currently being executed, schedule the
         * barrier to the worker; otherwise, put it after @target.
         */
        if (worker) {
                head = worker->scheduled.next;
                work_color = worker->current_color;
        } else {
                unsigned long *bits = work_data_bits(target);

                head = target->entry.next;
                /* there can already be other linked works, inherit and set */
                work_flags |= *bits & WORK_STRUCT_LINKED;
                work_color = get_work_color(*bits);
                __set_bit(WORK_STRUCT_LINKED_BIT, bits);
        }

        pwq->nr_in_flight[work_color]++;
        work_flags |= work_color_to_flags(work_color);

        insert_work(pwq, &barr->work, head, work_flags);
}

/**
 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
 * @wq: workqueue being flushed
 * @flush_color: new flush color, < 0 for no-op
 * @work_color: new work color, < 0 for no-op
 *
 * Prepare pwqs for workqueue flushing.
 *
 * If @flush_color is non-negative, flush_color on all pwqs should be
 * -1.  If no pwq has in-flight commands at the specified color, all
 * pwq->flush_color's stay at -1 and %false is returned.  If any pwq
 * has in flight commands, its pwq->flush_color is set to
 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
 * wakeup logic is armed and %true is returned.
 *
 * The caller should have initialized @wq->first_flusher prior to
 * calling this function with non-negative @flush_color.  If
 * @flush_color is negative, no flush color update is done and %false
 * is returned.
 *
 * If @work_color is non-negative, all pwqs should have the same
 * work_color which is previous to @work_color and all will be
 * advanced to @work_color.
 *
 * CONTEXT:
 * mutex_lock(wq->mutex).
 *
 * Return:
 * %true if @flush_color >= 0 and there's something to flush.  %false
 * otherwise.
 */
static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
                                      int flush_color, int work_color)
{
        bool wait = false;
        struct pool_workqueue *pwq;
        struct worker_pool *current_pool = NULL;

        if (flush_color >= 0) {
                WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
                atomic_set(&wq->nr_pwqs_to_flush, 1);
        }

        /*
         * For unbound workqueue, pwqs will map to only a few pools.
         * Most of the time, pwqs within the same pool will be linked
         * sequentially to wq->pwqs by cpu index. So in the majority
         * of pwq iters, the pool is the same, only doing lock/unlock
         * if the pool has changed. This can largely reduce expensive
         * lock operations.
         */
        for_each_pwq(pwq, wq) {
                if (current_pool != pwq->pool) {
                        if (likely(current_pool))
                                raw_spin_unlock_irq(&current_pool->lock);
                        current_pool = pwq->pool;
                        raw_spin_lock_irq(&current_pool->lock);
                }

                if (flush_color >= 0) {
                        WARN_ON_ONCE(pwq->flush_color != -1);

                        if (pwq->nr_in_flight[flush_color]) {
                                pwq->flush_color = flush_color;
                                atomic_inc(&wq->nr_pwqs_to_flush);
                                wait = true;
                        }
                }

                if (work_color >= 0) {
                        WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
                        pwq->work_color = work_color;
                }

        }

        if (current_pool)
                raw_spin_unlock_irq(&current_pool->lock);

        if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
                complete(&wq->first_flusher->done);

        return wait;
}

static void touch_wq_lockdep_map(struct workqueue_struct *wq)
{
#ifdef CONFIG_LOCKDEP
        if (unlikely(!wq->lockdep_map))
                return;

        if (wq->flags & WQ_BH)
                local_bh_disable();

        lock_map_acquire(wq->lockdep_map);
        lock_map_release(wq->lockdep_map);

        if (wq->flags & WQ_BH)
                local_bh_enable();
#endif
}

static void touch_work_lockdep_map(struct work_struct *work,
                                   struct workqueue_struct *wq)
{
#ifdef CONFIG_LOCKDEP
        if (wq->flags & WQ_BH)
                local_bh_disable();

        lock_map_acquire(&work->lockdep_map);
        lock_map_release(&work->lockdep_map);

        if (wq->flags & WQ_BH)
                local_bh_enable();
#endif
}

/**
 * __flush_workqueue - ensure that any scheduled work has run to completion.
 * @wq: workqueue to flush
 *
 * This function sleeps until all work items which were queued on entry
 * have finished execution, but it is not livelocked by new incoming ones.
 */
void __flush_workqueue(struct workqueue_struct *wq)
{
        struct wq_flusher this_flusher = {
                .list = LIST_HEAD_INIT(this_flusher.list),
                .flush_color = -1,
                .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, (*wq->lockdep_map)),
        };
        int next_color;

        if (WARN_ON(!wq_online))
                return;

        touch_wq_lockdep_map(wq);

        mutex_lock(&wq->mutex);

        /*
         * Start-to-wait phase
         */
        next_color = work_next_color(wq->work_color);

        if (next_color != wq->flush_color) {
                /*
                 * Color space is not full.  The current work_color
                 * becomes our flush_color and work_color is advanced
                 * by one.
                 */
                WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
                this_flusher.flush_color = wq->work_color;
                wq->work_color = next_color;

                if (!wq->first_flusher) {
                        /* no flush in progress, become the first flusher */
                        WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);

                        wq->first_flusher = &this_flusher;

                        if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
                                                       wq->work_color)) {
                                /* nothing to flush, done */
                                wq->flush_color = next_color;
                                wq->first_flusher = NULL;
                                goto out_unlock;
                        }
                } else {
                        /* wait in queue */
                        WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
                        list_add_tail(&this_flusher.list, &wq->flusher_queue);
                        flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
                }
        } else {
                /*
                 * Oops, color space is full, wait on overflow queue.
                 * The next flush completion will assign us
                 * flush_color and transfer to flusher_queue.
                 */
                list_add_tail(&this_flusher.list, &wq->flusher_overflow);
        }

        check_flush_dependency(wq, NULL);

        mutex_unlock(&wq->mutex);

        wait_for_completion(&this_flusher.done);

        /*
         * Wake-up-and-cascade phase
         *
         * First flushers are responsible for cascading flushes and
         * handling overflow.  Non-first flushers can simply return.
         */
        if (READ_ONCE(wq->first_flusher) != &this_flusher)
                return;

        mutex_lock(&wq->mutex);

        /* we might have raced, check again with mutex held */
        if (wq->first_flusher != &this_flusher)
                goto out_unlock;

        WRITE_ONCE(wq->first_flusher, NULL);

        WARN_ON_ONCE(!list_empty(&this_flusher.list));
        WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);

        while (true) {
                struct wq_flusher *next, *tmp;

                /* complete all the flushers sharing the current flush color */
                list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
                        if (next->flush_color != wq->flush_color)
                                break;
                        list_del_init(&next->list);
                        complete(&next->done);
                }

                WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
                             wq->flush_color != work_next_color(wq->work_color));

                /* this flush_color is finished, advance by one */
                wq->flush_color = work_next_color(wq->flush_color);

                /* one color has been freed, handle overflow queue */
                if (!list_empty(&wq->flusher_overflow)) {
                        /*
                         * Assign the same color to all overflowed
                         * flushers, advance work_color and append to
                         * flusher_queue.  This is the start-to-wait
                         * phase for these overflowed flushers.
                         */
                        list_for_each_entry(tmp, &wq->flusher_overflow, list)
                                tmp->flush_color = wq->work_color;

                        wq->work_color = work_next_color(wq->work_color);

                        list_splice_tail_init(&wq->flusher_overflow,
                                              &wq->flusher_queue);
                        flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
                }

                if (list_empty(&wq->flusher_queue)) {
                        WARN_ON_ONCE(wq->flush_color != wq->work_color);
                        break;
                }

                /*
                 * Need to flush more colors.  Make the next flusher
                 * the new first flusher and arm pwqs.
                 */
                WARN_ON_ONCE(wq->flush_color == wq->work_color);
                WARN_ON_ONCE(wq->flush_color != next->flush_color);

                list_del_init(&next->list);
                wq->first_flusher = next;

                if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
                        break;

                /*
                 * Meh... this color is already done, clear first
                 * flusher and repeat cascading.
                 */
                wq->first_flusher = NULL;
        }

out_unlock:
        mutex_unlock(&wq->mutex);
}
EXPORT_SYMBOL(__flush_workqueue);

/**
 * drain_workqueue - drain a workqueue
 * @wq: workqueue to drain
 *
 * Wait until the workqueue becomes empty.  While draining is in progress,
 * only chain queueing is allowed.  IOW, only currently pending or running
 * work items on @wq can queue further work items on it.  @wq is flushed
 * repeatedly until it becomes empty.  The number of flushing is determined
 * by the depth of chaining and should be relatively short.  Whine if it
 * takes too long.
 */
void drain_workqueue(struct workqueue_struct *wq)
{
        unsigned int flush_cnt = 0;
        struct pool_workqueue *pwq;

        /*
         * __queue_work() needs to test whether there are drainers, is much
         * hotter than drain_workqueue() and already looks at @wq->flags.
         * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
         */
        mutex_lock(&wq->mutex);
        if (!wq->nr_drainers++)
                wq->flags |= __WQ_DRAINING;
        mutex_unlock(&wq->mutex);
reflush:
        __flush_workqueue(wq);

        mutex_lock(&wq->mutex);

        for_each_pwq(pwq, wq) {
                bool drained;

                raw_spin_lock_irq(&pwq->pool->lock);
                drained = pwq_is_empty(pwq);
                raw_spin_unlock_irq(&pwq->pool->lock);

                if (drained)
                        continue;

                if (++flush_cnt == 10 ||
                    (flush_cnt % 100 == 0 && flush_cnt <= 1000))
                        pr_warn("workqueue %s: %s() isn't complete after %u tries\n",
                                wq->name, __func__, flush_cnt);

                mutex_unlock(&wq->mutex);
                goto reflush;
        }

        if (!--wq->nr_drainers)
                wq->flags &= ~__WQ_DRAINING;
        mutex_unlock(&wq->mutex);
}
EXPORT_SYMBOL_GPL(drain_workqueue);

static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
                             bool from_cancel)
{
        struct worker *worker = NULL;
        struct worker_pool *pool;
        struct pool_workqueue *pwq;
        struct workqueue_struct *wq;

        rcu_read_lock();
        pool = get_work_pool(work);
        if (!pool) {
                rcu_read_unlock();
                return false;
        }

        raw_spin_lock_irq(&pool->lock);
        /* see the comment in try_to_grab_pending() with the same code */
        pwq = get_work_pwq(work);
        if (pwq) {
                if (unlikely(pwq->pool != pool))
                        goto already_gone;
        } else {
                worker = find_worker_executing_work(pool, work);
                if (!worker)
                        goto already_gone;
                pwq = worker->current_pwq;
        }

        wq = pwq->wq;
        check_flush_dependency(wq, work);

        insert_wq_barrier(pwq, barr, work, worker);
        raw_spin_unlock_irq(&pool->lock);

        touch_work_lockdep_map(work, wq);

        /*
         * Force a lock recursion deadlock when using flush_work() inside a
         * single-threaded or rescuer equipped workqueue.
         *
         * For single threaded workqueues the deadlock happens when the work
         * is after the work issuing the flush_work(). For rescuer equipped
         * workqueues the deadlock happens when the rescuer stalls, blocking
         * forward progress.
         */
        if (!from_cancel && (wq->saved_max_active == 1 || wq->rescuer))
                touch_wq_lockdep_map(wq);

        rcu_read_unlock();
        return true;
already_gone:
        raw_spin_unlock_irq(&pool->lock);
        rcu_read_unlock();
        return false;
}

static bool __flush_work(struct work_struct *work, bool from_cancel)
{
        struct wq_barrier barr;

        if (WARN_ON(!wq_online))
                return false;

        if (WARN_ON(!work->func))
                return false;

        if (!start_flush_work(work, &barr, from_cancel))
                return false;

        /*
         * start_flush_work() returned %true. If @from_cancel is set, we know
         * that @work must have been executing during start_flush_work() and
         * can't currently be queued. Its data must contain OFFQ bits. If @work
         * was queued on a BH workqueue, we also know that it was running in the
         * BH context and thus can be busy-waited.
         */
        if (from_cancel) {
                unsigned long data = *work_data_bits(work);

                if (!WARN_ON_ONCE(data & WORK_STRUCT_PWQ) &&
                    (data & WORK_OFFQ_BH)) {
                        /*
                         * On RT, prevent a live lock when %current preempted
                         * soft interrupt processing or prevents ksoftirqd from
                         * running by keeping flipping BH. If the BH work item
                         * runs on a different CPU then this has no effect other
                         * than doing the BH disable/enable dance for nothing.
                         * This is copied from
                         * kernel/softirq.c::tasklet_unlock_spin_wait().
                         */
                        while (!try_wait_for_completion(&barr.done)) {
                                if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
                                        local_bh_disable();
                                        local_bh_enable();
                                } else {
                                        cpu_relax();
                                }
                        }
                        goto out_destroy;
                }
        }

        wait_for_completion(&barr.done);

out_destroy:
        destroy_work_on_stack(&barr.work);
        return true;
}

/**
 * flush_work - wait for a work to finish executing the last queueing instance
 * @work: the work to flush
 *
 * Wait until @work has finished execution.  @work is guaranteed to be idle
 * on return if it hasn't been requeued since flush started.
 *
 * Return:
 * %true if flush_work() waited for the work to finish execution,
 * %false if it was already idle.
 */
bool flush_work(struct work_struct *work)
{
        might_sleep();
        return __flush_work(work, false);
}
EXPORT_SYMBOL_GPL(flush_work);

/**
 * flush_delayed_work - wait for a dwork to finish executing the last queueing
 * @dwork: the delayed work to flush
 *
 * Delayed timer is cancelled and the pending work is queued for
 * immediate execution.  Like flush_work(), this function only
 * considers the last queueing instance of @dwork.
 *
 * Return:
 * %true if flush_work() waited for the work to finish execution,
 * %false if it was already idle.
 */
bool flush_delayed_work(struct delayed_work *dwork)
{
        local_irq_disable();
        if (del_timer_sync(&dwork->timer))
                __queue_work(dwork->cpu, dwork->wq, &dwork->work);
        local_irq_enable();
        return flush_work(&dwork->work);
}
EXPORT_SYMBOL(flush_delayed_work);

/**
 * flush_rcu_work - wait for a rwork to finish executing the last queueing
 * @rwork: the rcu work to flush
 *
 * Return:
 * %true if flush_rcu_work() waited for the work to finish execution,
 * %false if it was already idle.
 */
bool flush_rcu_work(struct rcu_work *rwork)
{
        if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
                rcu_barrier();
                flush_work(&rwork->work);
                return true;
        } else {
                return flush_work(&rwork->work);
        }
}
EXPORT_SYMBOL(flush_rcu_work);

static void work_offqd_disable(struct work_offq_data *offqd)
{
        const unsigned long max = (1lu << WORK_OFFQ_DISABLE_BITS) - 1;

        if (likely(offqd->disable < max))
                offqd->disable++;
        else
                WARN_ONCE(true, "workqueue: work disable count overflowed\n");
}

static void work_offqd_enable(struct work_offq_data *offqd)
{
        if (likely(offqd->disable > 0))
                offqd->disable--;
        else
                WARN_ONCE(true, "workqueue: work disable count underflowed\n");
}

static bool __cancel_work(struct work_struct *work, u32 cflags)
{
        struct work_offq_data offqd;
        unsigned long irq_flags;
        int ret;

        ret = work_grab_pending(work, cflags, &irq_flags);

        work_offqd_unpack(&offqd, *work_data_bits(work));

        if (cflags & WORK_CANCEL_DISABLE)
                work_offqd_disable(&offqd);

        set_work_pool_and_clear_pending(work, offqd.pool_id,
                                        work_offqd_pack_flags(&offqd));
        local_irq_restore(irq_flags);
        return ret;
}

static bool __cancel_work_sync(struct work_struct *work, u32 cflags)
{
        bool ret;

        ret = __cancel_work(work, cflags | WORK_CANCEL_DISABLE);

        if (*work_data_bits(work) & WORK_OFFQ_BH)
                WARN_ON_ONCE(in_hardirq());
        else
                might_sleep();

        /*
         * Skip __flush_work() during early boot when we know that @work isn't
         * executing. This allows canceling during early boot.
         */
        if (wq_online)
                __flush_work(work, true);

        if (!(cflags & WORK_CANCEL_DISABLE))
                enable_work(work);

        return ret;
}

/*
 * See cancel_delayed_work()
 */
bool cancel_work(struct work_struct *work)
{
        return __cancel_work(work, 0);
}
EXPORT_SYMBOL(cancel_work);

/**
 * cancel_work_sync - cancel a work and wait for it to finish
 * @work: the work to cancel
 *
 * Cancel @work and wait for its execution to finish. This function can be used
 * even if the work re-queues itself or migrates to another workqueue. On return
 * from this function, @work is guaranteed to be not pending or executing on any
 * CPU as long as there aren't racing enqueues.
 *
 * cancel_work_sync(&delayed_work->work) must not be used for delayed_work's.
 * Use cancel_delayed_work_sync() instead.
 *
 * Must be called from a sleepable context if @work was last queued on a non-BH
 * workqueue. Can also be called from non-hardirq atomic contexts including BH
 * if @work was last queued on a BH workqueue.
 *
 * Returns %true if @work was pending, %false otherwise.
 */
bool cancel_work_sync(struct work_struct *work)
{
        return __cancel_work_sync(work, 0);
}
EXPORT_SYMBOL_GPL(cancel_work_sync);

/**
 * cancel_delayed_work - cancel a delayed work
 * @dwork: delayed_work to cancel
 *
 * Kill off a pending delayed_work.
 *
 * Return: %true if @dwork was pending and canceled; %false if it wasn't
 * pending.
 *
 * Note:
 * The work callback function may still be running on return, unless
 * it returns %true and the work doesn't re-arm itself.  Explicitly flush or
 * use cancel_delayed_work_sync() to wait on it.
 *
 * This function is safe to call from any context including IRQ handler.
 */
bool cancel_delayed_work(struct delayed_work *dwork)
{
        return __cancel_work(&dwork->work, WORK_CANCEL_DELAYED);
}
EXPORT_SYMBOL(cancel_delayed_work);

/**
 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
 * @dwork: the delayed work cancel
 *
 * This is cancel_work_sync() for delayed works.
 *
 * Return:
 * %true if @dwork was pending, %false otherwise.
 */
bool cancel_delayed_work_sync(struct delayed_work *dwork)
{
        return __cancel_work_sync(&dwork->work, WORK_CANCEL_DELAYED);
}
EXPORT_SYMBOL(cancel_delayed_work_sync);

/**
 * disable_work - Disable and cancel a work item
 * @work: work item to disable
 *
 * Disable @work by incrementing its disable count and cancel it if currently
 * pending. As long as the disable count is non-zero, any attempt to queue @work
 * will fail and return %false. The maximum supported disable depth is 2 to the
 * power of %WORK_OFFQ_DISABLE_BITS, currently 65536.
 *
 * Can be called from any context. Returns %true if @work was pending, %false
 * otherwise.
 */
bool disable_work(struct work_struct *work)
{
        return __cancel_work(work, WORK_CANCEL_DISABLE);
}
EXPORT_SYMBOL_GPL(disable_work);

/**
 * disable_work_sync - Disable, cancel and drain a work item
 * @work: work item to disable
 *
 * Similar to disable_work() but also wait for @work to finish if currently
 * executing.
 *
 * Must be called from a sleepable context if @work was last queued on a non-BH
 * workqueue. Can also be called from non-hardirq atomic contexts including BH
 * if @work was last queued on a BH workqueue.
 *
 * Returns %true if @work was pending, %false otherwise.
 */
bool disable_work_sync(struct work_struct *work)
{
        return __cancel_work_sync(work, WORK_CANCEL_DISABLE);
}
EXPORT_SYMBOL_GPL(disable_work_sync);

/**
 * enable_work - Enable a work item
 * @work: work item to enable
 *
 * Undo disable_work[_sync]() by decrementing @work's disable count. @work can
 * only be queued if its disable count is 0.
 *
 * Can be called from any context. Returns %true if the disable count reached 0.
 * Otherwise, %false.
 */
bool enable_work(struct work_struct *work)
{
        struct work_offq_data offqd;
        unsigned long irq_flags;

        work_grab_pending(work, 0, &irq_flags);

        work_offqd_unpack(&offqd, *work_data_bits(work));
        work_offqd_enable(&offqd);
        set_work_pool_and_clear_pending(work, offqd.pool_id,
                                        work_offqd_pack_flags(&offqd));
        local_irq_restore(irq_flags);

        return !offqd.disable;
}
EXPORT_SYMBOL_GPL(enable_work);

/**
 * disable_delayed_work - Disable and cancel a delayed work item
 * @dwork: delayed work item to disable
 *
 * disable_work() for delayed work items.
 */
bool disable_delayed_work(struct delayed_work *dwork)
{
        return __cancel_work(&dwork->work,
                             WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE);
}
EXPORT_SYMBOL_GPL(disable_delayed_work);

/**
 * disable_delayed_work_sync - Disable, cancel and drain a delayed work item
 * @dwork: delayed work item to disable
 *
 * disable_work_sync() for delayed work items.
 */
bool disable_delayed_work_sync(struct delayed_work *dwork)
{
        return __cancel_work_sync(&dwork->work,
                                  WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE);
}
EXPORT_SYMBOL_GPL(disable_delayed_work_sync);

/**
 * enable_delayed_work - Enable a delayed work item
 * @dwork: delayed work item to enable
 *
 * enable_work() for delayed work items.
 */
bool enable_delayed_work(struct delayed_work *dwork)
{
        return enable_work(&dwork->work);
}
EXPORT_SYMBOL_GPL(enable_delayed_work);

/**
 * schedule_on_each_cpu - execute a function synchronously on each online CPU
 * @func: the function to call
 *
 * schedule_on_each_cpu() executes @func on each online CPU using the
 * system workqueue and blocks until all CPUs have completed.
 * schedule_on_each_cpu() is very slow.
 *
 * Return:
 * 0 on success, -errno on failure.
 */
int schedule_on_each_cpu(work_func_t func)
{
        int cpu;
        struct work_struct __percpu *works;

        works = alloc_percpu(struct work_struct);
        if (!works)
                return -ENOMEM;

        cpus_read_lock();

        for_each_online_cpu(cpu) {
                struct work_struct *work = per_cpu_ptr(works, cpu);

                INIT_WORK(work, func);
                schedule_work_on(cpu, work);
        }

        for_each_online_cpu(cpu)
                flush_work(per_cpu_ptr(works, cpu));

        cpus_read_unlock();
        free_percpu(works);
        return 0;
}

/**
 * execute_in_process_context - reliably execute the routine with user context
 * @fn:                the function to execute
 * @ew:                guaranteed storage for the execute work structure (must
 *                be available when the work executes)
 *
 * Executes the function immediately if process context is available,
 * otherwise schedules the function for delayed execution.
 *
 * Return:        0 - function was executed
 *                1 - function was scheduled for execution
 */
int execute_in_process_context(work_func_t fn, struct execute_work *ew)
{
        if (!in_interrupt()) {
                fn(&ew->work);
                return 0;
        }

        INIT_WORK(&ew->work, fn);
        schedule_work(&ew->work);

        return 1;
}
EXPORT_SYMBOL_GPL(execute_in_process_context);

/**
 * free_workqueue_attrs - free a workqueue_attrs
 * @attrs: workqueue_attrs to free
 *
 * Undo alloc_workqueue_attrs().
 */
void free_workqueue_attrs(struct workqueue_attrs *attrs)
{
        if (attrs) {
                free_cpumask_var(attrs->cpumask);
                free_cpumask_var(attrs->__pod_cpumask);
                kfree(attrs);
        }
}

/**
 * alloc_workqueue_attrs - allocate a workqueue_attrs
 *
 * Allocate a new workqueue_attrs, initialize with default settings and
 * return it.
 *
 * Return: The allocated new workqueue_attr on success. %NULL on failure.
 */
struct workqueue_attrs *alloc_workqueue_attrs(void)
{
        struct workqueue_attrs *attrs;

        attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
        if (!attrs)
                goto fail;
        if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
                goto fail;
        if (!alloc_cpumask_var(&attrs->__pod_cpumask, GFP_KERNEL))
                goto fail;

        cpumask_copy(attrs->cpumask, cpu_possible_mask);
        attrs->affn_scope = WQ_AFFN_DFL;
        return attrs;
fail:
        free_workqueue_attrs(attrs);
        return NULL;
}

static void copy_workqueue_attrs(struct workqueue_attrs *to,
                                 const struct workqueue_attrs *from)
{
        to->nice = from->nice;
        cpumask_copy(to->cpumask, from->cpumask);
        cpumask_copy(to->__pod_cpumask, from->__pod_cpumask);
        to->affn_strict = from->affn_strict;

        /*
         * Unlike hash and equality test, copying shouldn't ignore wq-only
         * fields as copying is used for both pool and wq attrs. Instead,
         * get_unbound_pool() explicitly clears the fields.
         */
        to->affn_scope = from->affn_scope;
        to->ordered = from->ordered;
}

/*
 * Some attrs fields are workqueue-only. Clear them for worker_pool's. See the
 * comments in 'struct workqueue_attrs' definition.
 */
static void wqattrs_clear_for_pool(struct workqueue_attrs *attrs)
{
        attrs->affn_scope = WQ_AFFN_NR_TYPES;
        attrs->ordered = false;
        if (attrs->affn_strict)
                cpumask_copy(attrs->cpumask, cpu_possible_mask);
}

/* hash value of the content of @attr */
static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
{
        u32 hash = 0;

        hash = jhash_1word(attrs->nice, hash);
        hash = jhash_1word(attrs->affn_strict, hash);
        hash = jhash(cpumask_bits(attrs->__pod_cpumask),
                     BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
        if (!attrs->affn_strict)
                hash = jhash(cpumask_bits(attrs->cpumask),
                             BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
        return hash;
}

/* content equality test */
static bool wqattrs_equal(const struct workqueue_attrs *a,
                          const struct workqueue_attrs *b)
{
        if (a->nice != b->nice)
                return false;
        if (a->affn_strict != b->affn_strict)
                return false;
        if (!cpumask_equal(a->__pod_cpumask, b->__pod_cpumask))
                return false;
        if (!a->affn_strict && !cpumask_equal(a->cpumask, b->cpumask))
                return false;
        return true;
}

/* Update @attrs with actually available CPUs */
static void wqattrs_actualize_cpumask(struct workqueue_attrs *attrs,
                                      const cpumask_t *unbound_cpumask)
{
        /*
         * Calculate the effective CPU mask of @attrs given @unbound_cpumask. If
         * @attrs->cpumask doesn't overlap with @unbound_cpumask, we fallback to
         * @unbound_cpumask.
         */
        cpumask_and(attrs->cpumask, attrs->cpumask, unbound_cpumask);
        if (unlikely(cpumask_empty(attrs->cpumask)))
                cpumask_copy(attrs->cpumask, unbound_cpumask);
}

/* find wq_pod_type to use for @attrs */
static const struct wq_pod_type *
wqattrs_pod_type(const struct workqueue_attrs *attrs)
{
        enum wq_affn_scope scope;
        struct wq_pod_type *pt;

        /* to synchronize access to wq_affn_dfl */
        lockdep_assert_held(&wq_pool_mutex);

        if (attrs->affn_scope == WQ_AFFN_DFL)
                scope = wq_affn_dfl;
        else
                scope = attrs->affn_scope;

        pt = &wq_pod_types[scope];

        if (!WARN_ON_ONCE(attrs->affn_scope == WQ_AFFN_NR_TYPES) &&
            likely(pt->nr_pods))
                return pt;

        /*
         * Before workqueue_init_topology(), only SYSTEM is available which is
         * initialized in workqueue_init_early().
         */
        pt = &wq_pod_types[WQ_AFFN_SYSTEM];
        BUG_ON(!pt->nr_pods);
        return pt;
}

/**
 * init_worker_pool - initialize a newly zalloc'd worker_pool
 * @pool: worker_pool to initialize
 *
 * Initialize a newly zalloc'd @pool.  It also allocates @pool->attrs.
 *
 * Return: 0 on success, -errno on failure.  Even on failure, all fields
 * inside @pool proper are initialized and put_unbound_pool() can be called
 * on @pool safely to release it.
 */
static int init_worker_pool(struct worker_pool *pool)
{
        raw_spin_lock_init(&pool->lock);
        pool->id = -1;
        pool->cpu = -1;
        pool->node = NUMA_NO_NODE;
        pool->flags |= POOL_DISASSOCIATED;
        pool->watchdog_ts = jiffies;
        INIT_LIST_HEAD(&pool->worklist);
        INIT_LIST_HEAD(&pool->idle_list);
        hash_init(pool->busy_hash);

        timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
        INIT_WORK(&pool->idle_cull_work, idle_cull_fn);

        timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);

        INIT_LIST_HEAD(&pool->workers);

        ida_init(&pool->worker_ida);
        INIT_HLIST_NODE(&pool->hash_node);
        pool->refcnt = 1;

        /* shouldn't fail above this point */
        pool->attrs = alloc_workqueue_attrs();
        if (!pool->attrs)
                return -ENOMEM;

        wqattrs_clear_for_pool(pool->attrs);

        return 0;
}

#ifdef CONFIG_LOCKDEP
static void wq_init_lockdep(struct workqueue_struct *wq)
{
        char *lock_name;

        lockdep_register_key(&wq->key);
        lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
        if (!lock_name)
                lock_name = wq->name;

        wq->lock_name = lock_name;
        wq->lockdep_map = &wq->__lockdep_map;
        lockdep_init_map(wq->lockdep_map, lock_name, &wq->key, 0);
}

static void wq_unregister_lockdep(struct workqueue_struct *wq)
{
        if (wq->lockdep_map != &wq->__lockdep_map)
                return;

        lockdep_unregister_key(&wq->key);
}

static void wq_free_lockdep(struct workqueue_struct *wq)
{
        if (wq->lockdep_map != &wq->__lockdep_map)
                return;

        if (wq->lock_name != wq->name)
                kfree(wq->lock_name);
}
#else
static void wq_init_lockdep(struct workqueue_struct *wq)
{
}

static void wq_unregister_lockdep(struct workqueue_struct *wq)
{
}

static void wq_free_lockdep(struct workqueue_struct *wq)
{
}
#endif

static void free_node_nr_active(struct wq_node_nr_active **nna_ar)
{
        int node;

        for_each_node(node) {
                kfree(nna_ar[node]);
                nna_ar[node] = NULL;
        }

        kfree(nna_ar[nr_node_ids]);
        nna_ar[nr_node_ids] = NULL;
}

static void init_node_nr_active(struct wq_node_nr_active *nna)
{
        nna->max = WQ_DFL_MIN_ACTIVE;
        atomic_set(&nna->nr, 0);
        raw_spin_lock_init(&nna->lock);
        INIT_LIST_HEAD(&nna->pending_pwqs);
}

/*
 * Each node's nr_active counter will be accessed mostly from its own node and
 * should be allocated in the node.
 */
static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar)
{
        struct wq_node_nr_active *nna;
        int node;

        for_each_node(node) {
                nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node);
                if (!nna)
                        goto err_free;
                init_node_nr_active(nna);
                nna_ar[node] = nna;
        }

        /* [nr_node_ids] is used as the fallback */
        nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE);
        if (!nna)
                goto err_free;
        init_node_nr_active(nna);
        nna_ar[nr_node_ids] = nna;

        return 0;

err_free:
        free_node_nr_active(nna_ar);
        return -ENOMEM;
}

static void rcu_free_wq(struct rcu_head *rcu)
{
        struct workqueue_struct *wq =
                container_of(rcu, struct workqueue_struct, rcu);

        if (wq->flags & WQ_UNBOUND)
                free_node_nr_active(wq->node_nr_active);

        wq_free_lockdep(wq);
        free_percpu(wq->cpu_pwq);
        free_workqueue_attrs(wq->unbound_attrs);
        kfree(wq);
}

static void rcu_free_pool(struct rcu_head *rcu)
{
        struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);

        ida_destroy(&pool->worker_ida);
        free_workqueue_attrs(pool->attrs);
        kfree(pool);
}

/**
 * put_unbound_pool - put a worker_pool
 * @pool: worker_pool to put
 *
 * Put @pool.  If its refcnt reaches zero, it gets destroyed in RCU
 * safe manner.  get_unbound_pool() calls this function on its failure path
 * and this function should be able to release pools which went through,
 * successfully or not, init_worker_pool().
 *
 * Should be called with wq_pool_mutex held.
 */
static void put_unbound_pool(struct worker_pool *pool)
{
        struct worker *worker;
        LIST_HEAD(cull_list);

        lockdep_assert_held(&wq_pool_mutex);

        if (--pool->refcnt)
                return;

        /* sanity checks */
        if (WARN_ON(!(pool->cpu < 0)) ||
            WARN_ON(!list_empty(&pool->worklist)))
                return;

        /* release id and unhash */
        if (pool->id >= 0)
                idr_remove(&worker_pool_idr, pool->id);
        hash_del(&pool->hash_node);

        /*
         * Become the manager and destroy all workers.  This prevents
         * @pool's workers from blocking on attach_mutex.  We're the last
         * manager and @pool gets freed with the flag set.
         *
         * Having a concurrent manager is quite unlikely to happen as we can
         * only get here with
         *   pwq->refcnt == pool->refcnt == 0
         * which implies no work queued to the pool, which implies no worker can
         * become the manager. However a worker could have taken the role of
         * manager before the refcnts dropped to 0, since maybe_create_worker()
         * drops pool->lock
         */
        while (true) {
                rcuwait_wait_event(&manager_wait,
                                   !(pool->flags & POOL_MANAGER_ACTIVE),
                                   TASK_UNINTERRUPTIBLE);

                mutex_lock(&wq_pool_attach_mutex);
                raw_spin_lock_irq(&pool->lock);
                if (!(pool->flags & POOL_MANAGER_ACTIVE)) {
                        pool->flags |= POOL_MANAGER_ACTIVE;
                        break;
                }
                raw_spin_unlock_irq(&pool->lock);
                mutex_unlock(&wq_pool_attach_mutex);
        }

        while ((worker = first_idle_worker(pool)))
                set_worker_dying(worker, &cull_list);
        WARN_ON(pool->nr_workers || pool->nr_idle);
        raw_spin_unlock_irq(&pool->lock);

        detach_dying_workers(&cull_list);

        mutex_unlock(&wq_pool_attach_mutex);

        reap_dying_workers(&cull_list);

        /* shut down the timers */
        del_timer_sync(&pool->idle_timer);
        cancel_work_sync(&pool->idle_cull_work);
        del_timer_sync(&pool->mayday_timer);

        /* RCU protected to allow dereferences from get_work_pool() */
        call_rcu(&pool->rcu, rcu_free_pool);
}

/**
 * get_unbound_pool - get a worker_pool with the specified attributes
 * @attrs: the attributes of the worker_pool to get
 *
 * Obtain a worker_pool which has the same attributes as @attrs, bump the
 * reference count and return it.  If there already is a matching
 * worker_pool, it will be used; otherwise, this function attempts to
 * create a new one.
 *
 * Should be called with wq_pool_mutex held.
 *
 * Return: On success, a worker_pool with the same attributes as @attrs.
 * On failure, %NULL.
 */
static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
{
        struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_NUMA];
        u32 hash = wqattrs_hash(attrs);
        struct worker_pool *pool;
        int pod, node = NUMA_NO_NODE;

        lockdep_assert_held(&wq_pool_mutex);

        /* do we already have a matching pool? */
        hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
                if (wqattrs_equal(pool->attrs, attrs)) {
                        pool->refcnt++;
                        return pool;
                }
        }

        /* If __pod_cpumask is contained inside a NUMA pod, that's our node */
        for (pod = 0; pod < pt->nr_pods; pod++) {
                if (cpumask_subset(attrs->__pod_cpumask, pt->pod_cpus[pod])) {
                        node = pt->pod_node[pod];
                        break;
                }
        }

        /* nope, create a new one */
        pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node);
        if (!pool || init_worker_pool(pool) < 0)
                goto fail;

        pool->node = node;
        copy_workqueue_attrs(pool->attrs, attrs);
        wqattrs_clear_for_pool(pool->attrs);

        if (worker_pool_assign_id(pool) < 0)
                goto fail;

        /* create and start the initial worker */
        if (wq_online && !create_worker(pool))
                goto fail;

        /* install */
        hash_add(unbound_pool_hash, &pool->hash_node, hash);

        return pool;
fail:
        if (pool)
                put_unbound_pool(pool);
        return NULL;
}

/*
 * Scheduled on pwq_release_worker by put_pwq() when an unbound pwq hits zero
 * refcnt and needs to be destroyed.
 */
static void pwq_release_workfn(struct kthread_work *work)
{
        struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
                                                  release_work);
        struct workqueue_struct *wq = pwq->wq;
        struct worker_pool *pool = pwq->pool;
        bool is_last = false;

        /*
         * When @pwq is not linked, it doesn't hold any reference to the
         * @wq, and @wq is invalid to access.
         */
        if (!list_empty(&pwq->pwqs_node)) {
                mutex_lock(&wq->mutex);
                list_del_rcu(&pwq->pwqs_node);
                is_last = list_empty(&wq->pwqs);

                /*
                 * For ordered workqueue with a plugged dfl_pwq, restart it now.
                 */
                if (!is_last && (wq->flags & __WQ_ORDERED))
                        unplug_oldest_pwq(wq);

                mutex_unlock(&wq->mutex);
        }

        if (wq->flags & WQ_UNBOUND) {
                mutex_lock(&wq_pool_mutex);
                put_unbound_pool(pool);
                mutex_unlock(&wq_pool_mutex);
        }

        if (!list_empty(&pwq->pending_node)) {
                struct wq_node_nr_active *nna =
                        wq_node_nr_active(pwq->wq, pwq->pool->node);

                raw_spin_lock_irq(&nna->lock);
                list_del_init(&pwq->pending_node);
                raw_spin_unlock_irq(&nna->lock);
        }

        kfree_rcu(pwq, rcu);

        /*
         * If we're the last pwq going away, @wq is already dead and no one
         * is gonna access it anymore.  Schedule RCU free.
         */
        if (is_last) {
                wq_unregister_lockdep(wq);
                call_rcu(&wq->rcu, rcu_free_wq);
        }
}

/* initialize newly allocated @pwq which is associated with @wq and @pool */
static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
                     struct worker_pool *pool)
{
        BUG_ON((unsigned long)pwq & ~WORK_STRUCT_PWQ_MASK);

        memset(pwq, 0, sizeof(*pwq));

        pwq->pool = pool;
        pwq->wq = wq;
        pwq->flush_color = -1;
        pwq->refcnt = 1;
        INIT_LIST_HEAD(&pwq->inactive_works);
        INIT_LIST_HEAD(&pwq->pending_node);
        INIT_LIST_HEAD(&pwq->pwqs_node);
        INIT_LIST_HEAD(&pwq->mayday_node);
        kthread_init_work(&pwq->release_work, pwq_release_workfn);
}

/* sync @pwq with the current state of its associated wq and link it */
static void link_pwq(struct pool_workqueue *pwq)
{
        struct workqueue_struct *wq = pwq->wq;

        lockdep_assert_held(&wq->mutex);

        /* may be called multiple times, ignore if already linked */
        if (!list_empty(&pwq->pwqs_node))
                return;

        /* set the matching work_color */
        pwq->work_color = wq->work_color;

        /* link in @pwq */
        list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs);
}

/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
                                        const struct workqueue_attrs *attrs)
{
        struct worker_pool *pool;
        struct pool_workqueue *pwq;

        lockdep_assert_held(&wq_pool_mutex);

        pool = get_unbound_pool(attrs);
        if (!pool)
                return NULL;

        pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
        if (!pwq) {
                put_unbound_pool(pool);
                return NULL;
        }

        init_pwq(pwq, wq, pool);
        return pwq;
}

static void apply_wqattrs_lock(void)
{
        mutex_lock(&wq_pool_mutex);
}

static void apply_wqattrs_unlock(void)
{
        mutex_unlock(&wq_pool_mutex);
}

/**
 * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod
 * @attrs: the wq_attrs of the default pwq of the target workqueue
 * @cpu: the target CPU
 *
 * Calculate the cpumask a workqueue with @attrs should use on @pod.
 * The result is stored in @attrs->__pod_cpumask.
 *
 * If pod affinity is not enabled, @attrs->cpumask is always used. If enabled
 * and @pod has online CPUs requested by @attrs, the returned cpumask is the
 * intersection of the possible CPUs of @pod and @attrs->cpumask.
 *
 * The caller is responsible for ensuring that the cpumask of @pod stays stable.
 */
static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu)
{
        const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
        int pod = pt->cpu_pod[cpu];

        /* calculate possible CPUs in @pod that @attrs wants */
        cpumask_and(attrs->__pod_cpumask, pt->pod_cpus[pod], attrs->cpumask);
        /* does @pod have any online CPUs @attrs wants? */
        if (!cpumask_intersects(attrs->__pod_cpumask, wq_online_cpumask)) {
                cpumask_copy(attrs->__pod_cpumask, attrs->cpumask);
                return;
        }
}

/* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
                                        int cpu, struct pool_workqueue *pwq)
{
        struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
        struct pool_workqueue *old_pwq;

        lockdep_assert_held(&wq_pool_mutex);
        lockdep_assert_held(&wq->mutex);

        /* link_pwq() can handle duplicate calls */
        link_pwq(pwq);

        old_pwq = rcu_access_pointer(*slot);
        rcu_assign_pointer(*slot, pwq);
        return old_pwq;
}

/* context to store the prepared attrs & pwqs before applying */
struct apply_wqattrs_ctx {
        struct workqueue_struct        *wq;                /* target workqueue */
        struct workqueue_attrs        *attrs;                /* attrs to apply */
        struct list_head        list;                /* queued for batching commit */
        struct pool_workqueue        *dfl_pwq;
        struct pool_workqueue        *pwq_tbl[];
};

/* free the resources after success or abort */
static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
{
        if (ctx) {
                int cpu;

                for_each_possible_cpu(cpu)
                        put_pwq_unlocked(ctx->pwq_tbl[cpu]);
                put_pwq_unlocked(ctx->dfl_pwq);

                free_workqueue_attrs(ctx->attrs);

                kfree(ctx);
        }
}

/* allocate the attrs and pwqs for later installation */
static struct apply_wqattrs_ctx *
apply_wqattrs_prepare(struct workqueue_struct *wq,
                      const struct workqueue_attrs *attrs,
                      const cpumask_var_t unbound_cpumask)
{
        struct apply_wqattrs_ctx *ctx;
        struct workqueue_attrs *new_attrs;
        int cpu;

        lockdep_assert_held(&wq_pool_mutex);

        if (WARN_ON(attrs->affn_scope < 0 ||
                    attrs->affn_scope >= WQ_AFFN_NR_TYPES))
                return ERR_PTR(-EINVAL);

        ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_cpu_ids), GFP_KERNEL);

        new_attrs = alloc_workqueue_attrs();
        if (!ctx || !new_attrs)
                goto out_free;

        /*
         * If something goes wrong during CPU up/down, we'll fall back to
         * the default pwq covering whole @attrs->cpumask.  Always create
         * it even if we don't use it immediately.
         */
        copy_workqueue_attrs(new_attrs, attrs);
        wqattrs_actualize_cpumask(new_attrs, unbound_cpumask);
        cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
        ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
        if (!ctx->dfl_pwq)
                goto out_free;

        for_each_possible_cpu(cpu) {
                if (new_attrs->ordered) {
                        ctx->dfl_pwq->refcnt++;
                        ctx->pwq_tbl[cpu] = ctx->dfl_pwq;
                } else {
                        wq_calc_pod_cpumask(new_attrs, cpu);
                        ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs);
                        if (!ctx->pwq_tbl[cpu])
                                goto out_free;
                }
        }

        /* save the user configured attrs and sanitize it. */
        copy_workqueue_attrs(new_attrs, attrs);
        cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
        cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
        ctx->attrs = new_attrs;

        /*
         * For initialized ordered workqueues, there should only be one pwq
         * (dfl_pwq). Set the plugged flag of ctx->dfl_pwq to suspend execution
         * of newly queued work items until execution of older work items in
         * the old pwq's have completed.
         */
        if ((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs))
                ctx->dfl_pwq->plugged = true;

        ctx->wq = wq;
        return ctx;

out_free:
        free_workqueue_attrs(new_attrs);
        apply_wqattrs_cleanup(ctx);
        return ERR_PTR(-ENOMEM);
}

/* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
{
        int cpu;

        /* all pwqs have been created successfully, let's install'em */
        mutex_lock(&ctx->wq->mutex);

        copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);

        /* save the previous pwqs and install the new ones */
        for_each_possible_cpu(cpu)
                ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
                                                        ctx->pwq_tbl[cpu]);
        ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);

        /* update node_nr_active->max */
        wq_update_node_max_active(ctx->wq, -1);

        /* rescuer needs to respect wq cpumask changes */
        if (ctx->wq->rescuer)
                set_cpus_allowed_ptr(ctx->wq->rescuer->task,
                                     unbound_effective_cpumask(ctx->wq));

        mutex_unlock(&ctx->wq->mutex);
}

static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
                                        const struct workqueue_attrs *attrs)
{
        struct apply_wqattrs_ctx *ctx;

        /* only unbound workqueues can change attributes */
        if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
                return -EINVAL;

        ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
        if (IS_ERR(ctx))
                return PTR_ERR(ctx);

        /* the ctx has been prepared successfully, let's commit it */
        apply_wqattrs_commit(ctx);
        apply_wqattrs_cleanup(ctx);

        return 0;
}

/**
 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
 * @wq: the target workqueue
 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
 *
 * Apply @attrs to an unbound workqueue @wq. Unless disabled, this function maps
 * a separate pwq to each CPU pod with possibles CPUs in @attrs->cpumask so that
 * work items are affine to the pod it was issued on. Older pwqs are released as
 * in-flight work items finish. Note that a work item which repeatedly requeues
 * itself back-to-back will stay on its current pwq.
 *
 * Performs GFP_KERNEL allocations.
 *
 * Return: 0 on success and -errno on failure.
 */
int apply_workqueue_attrs(struct workqueue_struct *wq,
                          const struct workqueue_attrs *attrs)
{
        int ret;

        mutex_lock(&wq_pool_mutex);
        ret = apply_workqueue_attrs_locked(wq, attrs);
        mutex_unlock(&wq_pool_mutex);

        return ret;
}

/**
 * unbound_wq_update_pwq - update a pwq slot for CPU hot[un]plug
 * @wq: the target workqueue
 * @cpu: the CPU to update the pwq slot for
 *
 * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
 * %CPU_DOWN_FAILED.  @cpu is in the same pod of the CPU being hot[un]plugged.
 *
 *
 * If pod affinity can't be adjusted due to memory allocation failure, it falls
 * back to @wq->dfl_pwq which may not be optimal but is always correct.
 *
 * Note that when the last allowed CPU of a pod goes offline for a workqueue
 * with a cpumask spanning multiple pods, the workers which were already
 * executing the work items for the workqueue will lose their CPU affinity and
 * may execute on any CPU. This is similar to how per-cpu workqueues behave on
 * CPU_DOWN. If a workqueue user wants strict affinity, it's the user's
 * responsibility to flush the work item from CPU_DOWN_PREPARE.
 */
static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
{
        struct pool_workqueue *old_pwq = NULL, *pwq;
        struct workqueue_attrs *target_attrs;

        lockdep_assert_held(&wq_pool_mutex);

        if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered)
                return;

        /*
         * We don't wanna alloc/free wq_attrs for each wq for each CPU.
         * Let's use a preallocated one.  The following buf is protected by
         * CPU hotplug exclusion.
         */
        target_attrs = unbound_wq_update_pwq_attrs_buf;

        copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
        wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask);

        /* nothing to do if the target cpumask matches the current pwq */
        wq_calc_pod_cpumask(target_attrs, cpu);
        if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
                return;

        /* create a new pwq */
        pwq = alloc_unbound_pwq(wq, target_attrs);
        if (!pwq) {
                pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",
                        wq->name);
                goto use_dfl_pwq;
        }

        /* Install the new pwq. */
        mutex_lock(&wq->mutex);
        old_pwq = install_unbound_pwq(wq, cpu, pwq);
        goto out_unlock;

use_dfl_pwq:
        mutex_lock(&wq->mutex);
        pwq = unbound_pwq(wq, -1);
        raw_spin_lock_irq(&pwq->pool->lock);
        get_pwq(pwq);
        raw_spin_unlock_irq(&pwq->pool->lock);
        old_pwq = install_unbound_pwq(wq, cpu, pwq);
out_unlock:
        mutex_unlock(&wq->mutex);
        put_pwq_unlocked(old_pwq);
}

static int alloc_and_link_pwqs(struct workqueue_struct *wq)
{
        bool highpri = wq->flags & WQ_HIGHPRI;
        int cpu, ret;

        lockdep_assert_held(&wq_pool_mutex);

        wq->cpu_pwq = alloc_percpu(struct pool_workqueue *);
        if (!wq->cpu_pwq)
                goto enomem;

        if (!(wq->flags & WQ_UNBOUND)) {
                struct worker_pool __percpu *pools;

                if (wq->flags & WQ_BH)
                        pools = bh_worker_pools;
                else
                        pools = cpu_worker_pools;

                for_each_possible_cpu(cpu) {
                        struct pool_workqueue **pwq_p;
                        struct worker_pool *pool;

                        pool = &(per_cpu_ptr(pools, cpu)[highpri]);
                        pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu);

                        *pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL,
                                                       pool->node);
                        if (!*pwq_p)
                                goto enomem;

                        init_pwq(*pwq_p, wq, pool);

                        mutex_lock(&wq->mutex);
                        link_pwq(*pwq_p);
                        mutex_unlock(&wq->mutex);
                }
                return 0;
        }

        if (wq->flags & __WQ_ORDERED) {
                struct pool_workqueue *dfl_pwq;

                ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]);
                /* there should only be single pwq for ordering guarantee */
                dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
                WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
                              wq->pwqs.prev != &dfl_pwq->pwqs_node),
                     "ordering guarantee broken for workqueue %s\n", wq->name);
        } else {
                ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]);
        }

        return ret;

enomem:
        if (wq->cpu_pwq) {
                for_each_possible_cpu(cpu) {
                        struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);

                        if (pwq)
                                kmem_cache_free(pwq_cache, pwq);
                }
                free_percpu(wq->cpu_pwq);
                wq->cpu_pwq = NULL;
        }
        return -ENOMEM;
}

static int wq_clamp_max_active(int max_active, unsigned int flags,
                               const char *name)
{
        if (max_active < 1 || max_active > WQ_MAX_ACTIVE)
                pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
                        max_active, name, 1, WQ_MAX_ACTIVE);

        return clamp_val(max_active, 1, WQ_MAX_ACTIVE);
}

/*
 * Workqueues which may be used during memory reclaim should have a rescuer
 * to guarantee forward progress.
 */
static int init_rescuer(struct workqueue_struct *wq)
{
        struct worker *rescuer;
        char id_buf[WORKER_ID_LEN];
        int ret;

        lockdep_assert_held(&wq_pool_mutex);

        if (!(wq->flags & WQ_MEM_RECLAIM))
                return 0;

        rescuer = alloc_worker(NUMA_NO_NODE);
        if (!rescuer) {
                pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",
                       wq->name);
                return -ENOMEM;
        }

        rescuer->rescue_wq = wq;
        format_worker_id(id_buf, sizeof(id_buf), rescuer, NULL);

        rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", id_buf);
        if (IS_ERR(rescuer->task)) {
                ret = PTR_ERR(rescuer->task);
                pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",
                       wq->name, ERR_PTR(ret));
                kfree(rescuer);
                return ret;
        }

        wq->rescuer = rescuer;
        if (wq->flags & WQ_UNBOUND)
                kthread_bind_mask(rescuer->task, unbound_effective_cpumask(wq));
        else
                kthread_bind_mask(rescuer->task, cpu_possible_mask);
        wake_up_process(rescuer->task);

        return 0;
}

/**
 * wq_adjust_max_active - update a wq's max_active to the current setting
 * @wq: target workqueue
 *
 * If @wq isn't freezing, set @wq->max_active to the saved_max_active and
 * activate inactive work items accordingly. If @wq is freezing, clear
 * @wq->max_active to zero.
 */
static void wq_adjust_max_active(struct workqueue_struct *wq)
{
        bool activated;
        int new_max, new_min;

        lockdep_assert_held(&wq->mutex);

        if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) {
                new_max = 0;
                new_min = 0;
        } else {
                new_max = wq->saved_max_active;
                new_min = wq->saved_min_active;
        }

        if (wq->max_active == new_max && wq->min_active == new_min)
                return;

        /*
         * Update @wq->max/min_active and then kick inactive work items if more
         * active work items are allowed. This doesn't break work item ordering
         * because new work items are always queued behind existing inactive
         * work items if there are any.
         */
        WRITE_ONCE(wq->max_active, new_max);
        WRITE_ONCE(wq->min_active, new_min);

        if (wq->flags & WQ_UNBOUND)
                wq_update_node_max_active(wq, -1);

        if (new_max == 0)
                return;

        /*
         * Round-robin through pwq's activating the first inactive work item
         * until max_active is filled.
         */
        do {
                struct pool_workqueue *pwq;

                activated = false;
                for_each_pwq(pwq, wq) {
                        unsigned long irq_flags;

                        /* can be called during early boot w/ irq disabled */
                        raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
                        if (pwq_activate_first_inactive(pwq, true)) {
                                activated = true;
                                kick_pool(pwq->pool);
                        }
                        raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
                }
        } while (activated);
}

static struct workqueue_struct *__alloc_workqueue(const char *fmt,
                                                  unsigned int flags,
                                                  int max_active, va_list args)
{
        struct workqueue_struct *wq;
        size_t wq_size;
        int name_len;

        if (flags & WQ_BH) {
                if (WARN_ON_ONCE(flags & ~__WQ_BH_ALLOWS))
                        return NULL;
                if (WARN_ON_ONCE(max_active))
                        return NULL;
        }

        /* see the comment above the definition of WQ_POWER_EFFICIENT */
        if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
                flags |= WQ_UNBOUND;

        /* allocate wq and format name */
        if (flags & WQ_UNBOUND)
                wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1);
        else
                wq_size = sizeof(*wq);

        wq = kzalloc(wq_size, GFP_KERNEL);
        if (!wq)
                return NULL;

        if (flags & WQ_UNBOUND) {
                wq->unbound_attrs = alloc_workqueue_attrs();
                if (!wq->unbound_attrs)
                        goto err_free_wq;
        }

        name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args);

        if (name_len >= WQ_NAME_LEN)
                pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n",
                             wq->name);

        if (flags & WQ_BH) {
                /*
                 * BH workqueues always share a single execution context per CPU
                 * and don't impose any max_active limit.
                 */
                max_active = INT_MAX;
        } else {
                max_active = max_active ?: WQ_DFL_ACTIVE;
                max_active = wq_clamp_max_active(max_active, flags, wq->name);
        }

        /* init wq */
        wq->flags = flags;
        wq->max_active = max_active;
        wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);
        wq->saved_max_active = wq->max_active;
        wq->saved_min_active = wq->min_active;
        mutex_init(&wq->mutex);
        atomic_set(&wq->nr_pwqs_to_flush, 0);
        INIT_LIST_HEAD(&wq->pwqs);
        INIT_LIST_HEAD(&wq->flusher_queue);
        INIT_LIST_HEAD(&wq->flusher_overflow);
        INIT_LIST_HEAD(&wq->maydays);

        INIT_LIST_HEAD(&wq->list);

        if (flags & WQ_UNBOUND) {
                if (alloc_node_nr_active(wq->node_nr_active) < 0)
                        goto err_free_wq;
        }

        /*
         * wq_pool_mutex protects the workqueues list, allocations of PWQs,
         * and the global freeze state.
         */
        apply_wqattrs_lock();

        if (alloc_and_link_pwqs(wq) < 0)
                goto err_unlock_free_node_nr_active;

        mutex_lock(&wq->mutex);
        wq_adjust_max_active(wq);
        mutex_unlock(&wq->mutex);

        list_add_tail_rcu(&wq->list, &workqueues);

        if (wq_online && init_rescuer(wq) < 0)
                goto err_unlock_destroy;

        apply_wqattrs_unlock();

        if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
                goto err_destroy;

        return wq;

err_unlock_free_node_nr_active:
        apply_wqattrs_unlock();
        /*
         * Failed alloc_and_link_pwqs() may leave pending pwq->release_work,
         * flushing the pwq_release_worker ensures that the pwq_release_workfn()
         * completes before calling kfree(wq).
         */
        if (wq->flags & WQ_UNBOUND) {
                kthread_flush_worker(pwq_release_worker);
                free_node_nr_active(wq->node_nr_active);
        }
err_free_wq:
        free_workqueue_attrs(wq->unbound_attrs);
        kfree(wq);
        return NULL;
err_unlock_destroy:
        apply_wqattrs_unlock();
err_destroy:
        destroy_workqueue(wq);
        return NULL;
}

__printf(1, 4)
struct workqueue_struct *alloc_workqueue(const char *fmt,
                                         unsigned int flags,
                                         int max_active, ...)
{
        struct workqueue_struct *wq;
        va_list args;

        va_start(args, max_active);
        wq = __alloc_workqueue(fmt, flags, max_active, args);
        va_end(args);
        if (!wq)
                return NULL;

        wq_init_lockdep(wq);

        return wq;
}
EXPORT_SYMBOL_GPL(alloc_workqueue);

#ifdef CONFIG_LOCKDEP
__printf(1, 5)
struct workqueue_struct *
alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags,
                            int max_active, struct lockdep_map *lockdep_map, ...)
{
        struct workqueue_struct *wq;
        va_list args;

        va_start(args, lockdep_map);
        wq = __alloc_workqueue(fmt, flags, max_active, args);
        va_end(args);
        if (!wq)
                return NULL;

        wq->lockdep_map = lockdep_map;

        return wq;
}
EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map);
#endif

static bool pwq_busy(struct pool_workqueue *pwq)
{
        int i;

        for (i = 0; i < WORK_NR_COLORS; i++)
                if (pwq->nr_in_flight[i])
                        return true;

        if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
                return true;
        if (!pwq_is_empty(pwq))
                return true;

        return false;
}

/**
 * destroy_workqueue - safely terminate a workqueue
 * @wq: target workqueue
 *
 * Safely destroy a workqueue. All work currently pending will be done first.
 */
void destroy_workqueue(struct workqueue_struct *wq)
{
        struct pool_workqueue *pwq;
        int cpu;

        /*
         * Remove it from sysfs first so that sanity check failure doesn't
         * lead to sysfs name conflicts.
         */
        workqueue_sysfs_unregister(wq);

        /* mark the workqueue destruction is in progress */
        mutex_lock(&wq->mutex);
        wq->flags |= __WQ_DESTROYING;
        mutex_unlock(&wq->mutex);

        /* drain it before proceeding with destruction */
        drain_workqueue(wq);

        /* kill rescuer, if sanity checks fail, leave it w/o rescuer */
        if (wq->rescuer) {
                struct worker *rescuer = wq->rescuer;

                /* this prevents new queueing */
                raw_spin_lock_irq(&wq_mayday_lock);
                wq->rescuer = NULL;
                raw_spin_unlock_irq(&wq_mayday_lock);

                /* rescuer will empty maydays list before exiting */
                kthread_stop(rescuer->task);
                kfree(rescuer);
        }

        /*
         * Sanity checks - grab all the locks so that we wait for all
         * in-flight operations which may do put_pwq().
         */
        mutex_lock(&wq_pool_mutex);
        mutex_lock(&wq->mutex);
        for_each_pwq(pwq, wq) {
                raw_spin_lock_irq(&pwq->pool->lock);
                if (WARN_ON(pwq_busy(pwq))) {
                        pr_warn("%s: %s has the following busy pwq\n",
                                __func__, wq->name);
                        show_pwq(pwq);
                        raw_spin_unlock_irq(&pwq->pool->lock);
                        mutex_unlock(&wq->mutex);
                        mutex_unlock(&wq_pool_mutex);
                        show_one_workqueue(wq);
                        return;
                }
                raw_spin_unlock_irq(&pwq->pool->lock);
        }
        mutex_unlock(&wq->mutex);

        /*
         * wq list is used to freeze wq, remove from list after
         * flushing is complete in case freeze races us.
         */
        list_del_rcu(&wq->list);
        mutex_unlock(&wq_pool_mutex);

        /*
         * We're the sole accessor of @wq. Directly access cpu_pwq and dfl_pwq
         * to put the base refs. @wq will be auto-destroyed from the last
         * pwq_put. RCU read lock prevents @wq from going away from under us.
         */
        rcu_read_lock();

        for_each_possible_cpu(cpu) {
                put_pwq_unlocked(unbound_pwq(wq, cpu));
                RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
        }

        put_pwq_unlocked(unbound_pwq(wq, -1));
        RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);

        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(destroy_workqueue);

/**
 * workqueue_set_max_active - adjust max_active of a workqueue
 * @wq: target workqueue
 * @max_active: new max_active value.
 *
 * Set max_active of @wq to @max_active. See the alloc_workqueue() function
 * comment.
 *
 * CONTEXT:
 * Don't call from IRQ context.
 */
void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
{
        /* max_active doesn't mean anything for BH workqueues */
        if (WARN_ON(wq->flags & WQ_BH))
                return;
        /* disallow meddling with max_active for ordered workqueues */
        if (WARN_ON(wq->flags & __WQ_ORDERED))
                return;

        max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);

        mutex_lock(&wq->mutex);

        wq->saved_max_active = max_active;
        if (wq->flags & WQ_UNBOUND)
                wq->saved_min_active = min(wq->saved_min_active, max_active);

        wq_adjust_max_active(wq);

        mutex_unlock(&wq->mutex);
}
EXPORT_SYMBOL_GPL(workqueue_set_max_active);

/**
 * workqueue_set_min_active - adjust min_active of an unbound workqueue
 * @wq: target unbound workqueue
 * @min_active: new min_active value
 *
 * Set min_active of an unbound workqueue. Unlike other types of workqueues, an
 * unbound workqueue is not guaranteed to be able to process max_active
 * interdependent work items. Instead, an unbound workqueue is guaranteed to be
 * able to process min_active number of interdependent work items which is
 * %WQ_DFL_MIN_ACTIVE by default.
 *
 * Use this function to adjust the min_active value between 0 and the current
 * max_active.
 */
void workqueue_set_min_active(struct workqueue_struct *wq, int min_active)
{
        /* min_active is only meaningful for non-ordered unbound workqueues */
        if (WARN_ON((wq->flags & (WQ_BH | WQ_UNBOUND | __WQ_ORDERED)) !=
                    WQ_UNBOUND))
                return;

        mutex_lock(&wq->mutex);
        wq->saved_min_active = clamp(min_active, 0, wq->saved_max_active);
        wq_adjust_max_active(wq);
        mutex_unlock(&wq->mutex);
}

/**
 * current_work - retrieve %current task's work struct
 *
 * Determine if %current task is a workqueue worker and what it's working on.
 * Useful to find out the context that the %current task is running in.
 *
 * Return: work struct if %current task is a workqueue worker, %NULL otherwise.
 */
struct work_struct *current_work(void)
{
        struct worker *worker = current_wq_worker();

        return worker ? worker->current_work : NULL;
}
EXPORT_SYMBOL(current_work);

/**
 * current_is_workqueue_rescuer - is %current workqueue rescuer?
 *
 * Determine whether %current is a workqueue rescuer.  Can be used from
 * work functions to determine whether it's being run off the rescuer task.
 *
 * Return: %true if %current is a workqueue rescuer. %false otherwise.
 */
bool current_is_workqueue_rescuer(void)
{
        struct worker *worker = current_wq_worker();

        return worker && worker->rescue_wq;
}

/**
 * workqueue_congested - test whether a workqueue is congested
 * @cpu: CPU in question
 * @wq: target workqueue
 *
 * Test whether @wq's cpu workqueue for @cpu is congested.  There is
 * no synchronization around this function and the test result is
 * unreliable and only useful as advisory hints or for debugging.
 *
 * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
 *
 * With the exception of ordered workqueues, all workqueues have per-cpu
 * pool_workqueues, each with its own congested state. A workqueue being
 * congested on one CPU doesn't mean that the workqueue is contested on any
 * other CPUs.
 *
 * Return:
 * %true if congested, %false otherwise.
 */
bool workqueue_congested(int cpu, struct workqueue_struct *wq)
{
        struct pool_workqueue *pwq;
        bool ret;

        rcu_read_lock();
        preempt_disable();

        if (cpu == WORK_CPU_UNBOUND)
                cpu = smp_processor_id();

        pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
        ret = !list_empty(&pwq->inactive_works);

        preempt_enable();
        rcu_read_unlock();

        return ret;
}
EXPORT_SYMBOL_GPL(workqueue_congested);

/**
 * work_busy - test whether a work is currently pending or running
 * @work: the work to be tested
 *
 * Test whether @work is currently pending or running.  There is no
 * synchronization around this function and the test result is
 * unreliable and only useful as advisory hints or for debugging.
 *
 * Return:
 * OR'd bitmask of WORK_BUSY_* bits.
 */
unsigned int work_busy(struct work_struct *work)
{
        struct worker_pool *pool;
        unsigned long irq_flags;
        unsigned int ret = 0;

        if (work_pending(work))
                ret |= WORK_BUSY_PENDING;

        rcu_read_lock();
        pool = get_work_pool(work);
        if (pool) {
                raw_spin_lock_irqsave(&pool->lock, irq_flags);
                if (find_worker_executing_work(pool, work))
                        ret |= WORK_BUSY_RUNNING;
                raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
        }
        rcu_read_unlock();

        return ret;
}
EXPORT_SYMBOL_GPL(work_busy);

/**
 * set_worker_desc - set description for the current work item
 * @fmt: printf-style format string
 * @...: arguments for the format string
 *
 * This function can be called by a running work function to describe what
 * the work item is about.  If the worker task gets dumped, this
 * information will be printed out together to help debugging.  The
 * description can be at most WORKER_DESC_LEN including the trailing '\0'.
 */
void set_worker_desc(const char *fmt, ...)
{
        struct worker *worker = current_wq_worker();
        va_list args;

        if (worker) {
                va_start(args, fmt);
                vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
                va_end(args);
        }
}
EXPORT_SYMBOL_GPL(set_worker_desc);

/**
 * print_worker_info - print out worker information and description
 * @log_lvl: the log level to use when printing
 * @task: target task
 *
 * If @task is a worker and currently executing a work item, print out the
 * name of the workqueue being serviced and worker description set with
 * set_worker_desc() by the currently executing work item.
 *
 * This function can be safely called on any task as long as the
 * task_struct itself is accessible.  While safe, this function isn't
 * synchronized and may print out mixups or garbages of limited length.
 */
void print_worker_info(const char *log_lvl, struct task_struct *task)
{
        work_func_t *fn = NULL;
        char name[WQ_NAME_LEN] = { };
        char desc[WORKER_DESC_LEN] = { };
        struct pool_workqueue *pwq = NULL;
        struct workqueue_struct *wq = NULL;
        struct worker *worker;

        if (!(task->flags & PF_WQ_WORKER))
                return;

        /*
         * This function is called without any synchronization and @task
         * could be in any state.  Be careful with dereferences.
         */
        worker = kthread_probe_data(task);

        /*
         * Carefully copy the associated workqueue's workfn, name and desc.
         * Keep the original last '\0' in case the original is garbage.
         */
        copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
        copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
        copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
        copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
        copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);

        if (fn || name[0] || desc[0]) {
                printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
                if (strcmp(name, desc))
                        pr_cont(" (%s)", desc);
                pr_cont("\n");
        }
}

static void pr_cont_pool_info(struct worker_pool *pool)
{
        pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
        if (pool->node != NUMA_NO_NODE)
                pr_cont(" node=%d", pool->node);
        pr_cont(" flags=0x%x", pool->flags);
        if (pool->flags & POOL_BH)
                pr_cont(" bh%s",
                        pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
        else
                pr_cont(" nice=%d", pool->attrs->nice);
}

static void pr_cont_worker_id(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;

        if (pool->flags & WQ_BH)
                pr_cont("bh%s",
                        pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
        else
                pr_cont("%d%s", task_pid_nr(worker->task),
                        worker->rescue_wq ? "(RESCUER)" : "");
}

struct pr_cont_work_struct {
        bool comma;
        work_func_t func;
        long ctr;
};

static void pr_cont_work_flush(bool comma, work_func_t func, struct pr_cont_work_struct *pcwsp)
{
        if (!pcwsp->ctr)
                goto out_record;
        if (func == pcwsp->func) {
                pcwsp->ctr++;
                return;
        }
        if (pcwsp->ctr == 1)
                pr_cont("%s %ps", pcwsp->comma ? "," : "", pcwsp->func);
        else
                pr_cont("%s %ld*%ps", pcwsp->comma ? "," : "", pcwsp->ctr, pcwsp->func);
        pcwsp->ctr = 0;
out_record:
        if ((long)func == -1L)
                return;
        pcwsp->comma = comma;
        pcwsp->func = func;
        pcwsp->ctr = 1;
}

static void pr_cont_work(bool comma, struct work_struct *work, struct pr_cont_work_struct *pcwsp)
{
        if (work->func == wq_barrier_func) {
                struct wq_barrier *barr;

                barr = container_of(work, struct wq_barrier, work);

                pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
                pr_cont("%s BAR(%d)", comma ? "," : "",
                        task_pid_nr(barr->task));
        } else {
                if (!comma)
                        pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
                pr_cont_work_flush(comma, work->func, pcwsp);
        }
}

static void show_pwq(struct pool_workqueue *pwq)
{
        struct pr_cont_work_struct pcws = { .ctr = 0, };
        struct worker_pool *pool = pwq->pool;
        struct work_struct *work;
        struct worker *worker;
        bool has_in_flight = false, has_pending = false;
        int bkt;

        pr_info("  pwq %d:", pool->id);
        pr_cont_pool_info(pool);

        pr_cont(" active=%d refcnt=%d%s\n",
                pwq->nr_active, pwq->refcnt,
                !list_empty(&pwq->mayday_node) ? " MAYDAY" : "");

        hash_for_each(pool->busy_hash, bkt, worker, hentry) {
                if (worker->current_pwq == pwq) {
                        has_in_flight = true;
                        break;
                }
        }
        if (has_in_flight) {
                bool comma = false;

                pr_info("    in-flight:");
                hash_for_each(pool->busy_hash, bkt, worker, hentry) {
                        if (worker->current_pwq != pwq)
                                continue;

                        pr_cont(" %s", comma ? "," : "");
                        pr_cont_worker_id(worker);
                        pr_cont(":%ps", worker->current_func);
                        list_for_each_entry(work, &worker->scheduled, entry)
                                pr_cont_work(false, work, &pcws);
                        pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
                        comma = true;
                }
                pr_cont("\n");
        }

        list_for_each_entry(work, &pool->worklist, entry) {
                if (get_work_pwq(work) == pwq) {
                        has_pending = true;
                        break;
                }
        }
        if (has_pending) {
                bool comma = false;

                pr_info("    pending:");
                list_for_each_entry(work, &pool->worklist, entry) {
                        if (get_work_pwq(work) != pwq)
                                continue;

                        pr_cont_work(comma, work, &pcws);
                        comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
                }
                pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
                pr_cont("\n");
        }

        if (!list_empty(&pwq->inactive_works)) {
                bool comma = false;

                pr_info("    inactive:");
                list_for_each_entry(work, &pwq->inactive_works, entry) {
                        pr_cont_work(comma, work, &pcws);
                        comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
                }
                pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
                pr_cont("\n");
        }
}

/**
 * show_one_workqueue - dump state of specified workqueue
 * @wq: workqueue whose state will be printed
 */
void show_one_workqueue(struct workqueue_struct *wq)
{
        struct pool_workqueue *pwq;
        bool idle = true;
        unsigned long irq_flags;

        for_each_pwq(pwq, wq) {
                if (!pwq_is_empty(pwq)) {
                        idle = false;
                        break;
                }
        }
        if (idle) /* Nothing to print for idle workqueue */
                return;

        pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);

        for_each_pwq(pwq, wq) {
                raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
                if (!pwq_is_empty(pwq)) {
                        /*
                         * Defer printing to avoid deadlocks in console
                         * drivers that queue work while holding locks
                         * also taken in their write paths.
                         */
                        printk_deferred_enter();
                        show_pwq(pwq);
                        printk_deferred_exit();
                }
                raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
                /*
                 * We could be printing a lot from atomic context, e.g.
                 * sysrq-t -> show_all_workqueues(). Avoid triggering
                 * hard lockup.
                 */
                touch_nmi_watchdog();
        }

}

/**
 * show_one_worker_pool - dump state of specified worker pool
 * @pool: worker pool whose state will be printed
 */
static void show_one_worker_pool(struct worker_pool *pool)
{
        struct worker *worker;
        bool first = true;
        unsigned long irq_flags;
        unsigned long hung = 0;

        raw_spin_lock_irqsave(&pool->lock, irq_flags);
        if (pool->nr_workers == pool->nr_idle)
                goto next_pool;

        /* How long the first pending work is waiting for a worker. */
        if (!list_empty(&pool->worklist))
                hung = jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000;

        /*
         * Defer printing to avoid deadlocks in console drivers that
         * queue work while holding locks also taken in their write
         * paths.
         */
        printk_deferred_enter();
        pr_info("pool %d:", pool->id);
        pr_cont_pool_info(pool);
        pr_cont(" hung=%lus workers=%d", hung, pool->nr_workers);
        if (pool->manager)
                pr_cont(" manager: %d",
                        task_pid_nr(pool->manager->task));
        list_for_each_entry(worker, &pool->idle_list, entry) {
                pr_cont(" %s", first ? "idle: " : "");
                pr_cont_worker_id(worker);
                first = false;
        }
        pr_cont("\n");
        printk_deferred_exit();
next_pool:
        raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
        /*
         * We could be printing a lot from atomic context, e.g.
         * sysrq-t -> show_all_workqueues(). Avoid triggering
         * hard lockup.
         */
        touch_nmi_watchdog();

}

/**
 * show_all_workqueues - dump workqueue state
 *
 * Called from a sysrq handler and prints out all busy workqueues and pools.
 */
void show_all_workqueues(void)
{
        struct workqueue_struct *wq;
        struct worker_pool *pool;
        int pi;

        rcu_read_lock();

        pr_info("Showing busy workqueues and worker pools:\n");

        list_for_each_entry_rcu(wq, &workqueues, list)
                show_one_workqueue(wq);

        for_each_pool(pool, pi)
                show_one_worker_pool(pool);

        rcu_read_unlock();
}

/**
 * show_freezable_workqueues - dump freezable workqueue state
 *
 * Called from try_to_freeze_tasks() and prints out all freezable workqueues
 * still busy.
 */
void show_freezable_workqueues(void)
{
        struct workqueue_struct *wq;

        rcu_read_lock();

        pr_info("Showing freezable workqueues that are still busy:\n");

        list_for_each_entry_rcu(wq, &workqueues, list) {
                if (!(wq->flags & WQ_FREEZABLE))
                        continue;
                show_one_workqueue(wq);
        }

        rcu_read_unlock();
}

/* used to show worker information through /proc/PID/{comm,stat,status} */
void wq_worker_comm(char *buf, size_t size, struct task_struct *task)
{
        /* stabilize PF_WQ_WORKER and worker pool association */
        mutex_lock(&wq_pool_attach_mutex);

        if (task->flags & PF_WQ_WORKER) {
                struct worker *worker = kthread_data(task);
                struct worker_pool *pool = worker->pool;
                int off;

                off = format_worker_id(buf, size, worker, pool);

                if (pool) {
                        raw_spin_lock_irq(&pool->lock);
                        /*
                         * ->desc tracks information (wq name or
                         * set_worker_desc()) for the latest execution.  If
                         * current, prepend '+', otherwise '-'.
                         */
                        if (worker->desc[0] != '\0') {
                                if (worker->current_work)
                                        scnprintf(buf + off, size - off, "+%s",
                                                  worker->desc);
                                else
                                        scnprintf(buf + off, size - off, "-%s",
                                                  worker->desc);
                        }
                        raw_spin_unlock_irq(&pool->lock);
                }
        } else {
                strscpy(buf, task->comm, size);
        }

        mutex_unlock(&wq_pool_attach_mutex);
}

#ifdef CONFIG_SMP

/*
 * CPU hotplug.
 *
 * There are two challenges in supporting CPU hotplug.  Firstly, there
 * are a lot of assumptions on strong associations among work, pwq and
 * pool which make migrating pending and scheduled works very
 * difficult to implement without impacting hot paths.  Secondly,
 * worker pools serve mix of short, long and very long running works making
 * blocked draining impractical.
 *
 * This is solved by allowing the pools to be disassociated from the CPU
 * running as an unbound one and allowing it to be reattached later if the
 * cpu comes back online.
 */

static void unbind_workers(int cpu)
{
        struct worker_pool *pool;
        struct worker *worker;

        for_each_cpu_worker_pool(pool, cpu) {
                mutex_lock(&wq_pool_attach_mutex);
                raw_spin_lock_irq(&pool->lock);

                /*
                 * We've blocked all attach/detach operations. Make all workers
                 * unbound and set DISASSOCIATED.  Before this, all workers
                 * must be on the cpu.  After this, they may become diasporas.
                 * And the preemption disabled section in their sched callbacks
                 * are guaranteed to see WORKER_UNBOUND since the code here
                 * is on the same cpu.
                 */
                for_each_pool_worker(worker, pool)
                        worker->flags |= WORKER_UNBOUND;

                pool->flags |= POOL_DISASSOCIATED;

                /*
                 * The handling of nr_running in sched callbacks are disabled
                 * now.  Zap nr_running.  After this, nr_running stays zero and
                 * need_more_worker() and keep_working() are always true as
                 * long as the worklist is not empty.  This pool now behaves as
                 * an unbound (in terms of concurrency management) pool which
                 * are served by workers tied to the pool.
                 */
                pool->nr_running = 0;

                /*
                 * With concurrency management just turned off, a busy
                 * worker blocking could lead to lengthy stalls.  Kick off
                 * unbound chain execution of currently pending work items.
                 */
                kick_pool(pool);

                raw_spin_unlock_irq(&pool->lock);

                for_each_pool_worker(worker, pool)
                        unbind_worker(worker);

                mutex_unlock(&wq_pool_attach_mutex);
        }
}

/**
 * rebind_workers - rebind all workers of a pool to the associated CPU
 * @pool: pool of interest
 *
 * @pool->cpu is coming online.  Rebind all workers to the CPU.
 */
static void rebind_workers(struct worker_pool *pool)
{
        struct worker *worker;

        lockdep_assert_held(&wq_pool_attach_mutex);

        /*
         * Restore CPU affinity of all workers.  As all idle workers should
         * be on the run-queue of the associated CPU before any local
         * wake-ups for concurrency management happen, restore CPU affinity
         * of all workers first and then clear UNBOUND.  As we're called
         * from CPU_ONLINE, the following shouldn't fail.
         */
        for_each_pool_worker(worker, pool) {
                kthread_set_per_cpu(worker->task, pool->cpu);
                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
                                                  pool_allowed_cpus(pool)) < 0);
        }

        raw_spin_lock_irq(&pool->lock);

        pool->flags &= ~POOL_DISASSOCIATED;

        for_each_pool_worker(worker, pool) {
                unsigned int worker_flags = worker->flags;

                /*
                 * We want to clear UNBOUND but can't directly call
                 * worker_clr_flags() or adjust nr_running.  Atomically
                 * replace UNBOUND with another NOT_RUNNING flag REBOUND.
                 * @worker will clear REBOUND using worker_clr_flags() when
                 * it initiates the next execution cycle thus restoring
                 * concurrency management.  Note that when or whether
                 * @worker clears REBOUND doesn't affect correctness.
                 *
                 * WRITE_ONCE() is necessary because @worker->flags may be
                 * tested without holding any lock in
                 * wq_worker_running().  Without it, NOT_RUNNING test may
                 * fail incorrectly leading to premature concurrency
                 * management operations.
                 */
                WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
                worker_flags |= WORKER_REBOUND;
                worker_flags &= ~WORKER_UNBOUND;
                WRITE_ONCE(worker->flags, worker_flags);
        }

        raw_spin_unlock_irq(&pool->lock);
}

/**
 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
 * @pool: unbound pool of interest
 * @cpu: the CPU which is coming up
 *
 * An unbound pool may end up with a cpumask which doesn't have any online
 * CPUs.  When a worker of such pool get scheduled, the scheduler resets
 * its cpus_allowed.  If @cpu is in @pool's cpumask which didn't have any
 * online CPU before, cpus_allowed of all its workers should be restored.
 */
static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
{
        static cpumask_t cpumask;
        struct worker *worker;

        lockdep_assert_held(&wq_pool_attach_mutex);

        /* is @cpu allowed for @pool? */
        if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
                return;

        cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);

        /* as we're called from CPU_ONLINE, the following shouldn't fail */
        for_each_pool_worker(worker, pool)
                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
}

int workqueue_prepare_cpu(unsigned int cpu)
{
        struct worker_pool *pool;

        for_each_cpu_worker_pool(pool, cpu) {
                if (pool->nr_workers)
                        continue;
                if (!create_worker(pool))
                        return -ENOMEM;
        }
        return 0;
}

int workqueue_online_cpu(unsigned int cpu)
{
        struct worker_pool *pool;
        struct workqueue_struct *wq;
        int pi;

        mutex_lock(&wq_pool_mutex);

        cpumask_set_cpu(cpu, wq_online_cpumask);

        for_each_pool(pool, pi) {
                /* BH pools aren't affected by hotplug */
                if (pool->flags & POOL_BH)
                        continue;

                mutex_lock(&wq_pool_attach_mutex);
                if (pool->cpu == cpu)
                        rebind_workers(pool);
                else if (pool->cpu < 0)
                        restore_unbound_workers_cpumask(pool, cpu);
                mutex_unlock(&wq_pool_attach_mutex);
        }

        /* update pod affinity of unbound workqueues */
        list_for_each_entry(wq, &workqueues, list) {
                struct workqueue_attrs *attrs = wq->unbound_attrs;

                if (attrs) {
                        const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
                        int tcpu;

                        for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]])
                                unbound_wq_update_pwq(wq, tcpu);

                        mutex_lock(&wq->mutex);
                        wq_update_node_max_active(wq, -1);
                        mutex_unlock(&wq->mutex);
                }
        }

        mutex_unlock(&wq_pool_mutex);
        return 0;
}

int workqueue_offline_cpu(unsigned int cpu)
{
        struct workqueue_struct *wq;

        /* unbinding per-cpu workers should happen on the local CPU */
        if (WARN_ON(cpu != smp_processor_id()))
                return -1;

        unbind_workers(cpu);

        /* update pod affinity of unbound workqueues */
        mutex_lock(&wq_pool_mutex);

        cpumask_clear_cpu(cpu, wq_online_cpumask);

        list_for_each_entry(wq, &workqueues, list) {
                struct workqueue_attrs *attrs = wq->unbound_attrs;

                if (attrs) {
                        const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
                        int tcpu;

                        for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]])
                                unbound_wq_update_pwq(wq, tcpu);

                        mutex_lock(&wq->mutex);
                        wq_update_node_max_active(wq, cpu);
                        mutex_unlock(&wq->mutex);
                }
        }
        mutex_unlock(&wq_pool_mutex);

        return 0;
}

struct work_for_cpu {
        struct work_struct work;
        long (*fn)(void *);
        void *arg;
        long ret;
};

static void work_for_cpu_fn(struct work_struct *work)
{
        struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);

        wfc->ret = wfc->fn(wfc->arg);
}

/**
 * work_on_cpu_key - run a function in thread context on a particular cpu
 * @cpu: the cpu to run on
 * @fn: the function to run
 * @arg: the function arg
 * @key: The lock class key for lock debugging purposes
 *
 * It is up to the caller to ensure that the cpu doesn't go offline.
 * The caller must not hold any locks which would prevent @fn from completing.
 *
 * Return: The value @fn returns.
 */
long work_on_cpu_key(int cpu, long (*fn)(void *),
                     void *arg, struct lock_class_key *key)
{
        struct work_for_cpu wfc = { .fn = fn, .arg = arg };

        INIT_WORK_ONSTACK_KEY(&wfc.work, work_for_cpu_fn, key);
        schedule_work_on(cpu, &wfc.work);
        flush_work(&wfc.work);
        destroy_work_on_stack(&wfc.work);
        return wfc.ret;
}
EXPORT_SYMBOL_GPL(work_on_cpu_key);

/**
 * work_on_cpu_safe_key - run a function in thread context on a particular cpu
 * @cpu: the cpu to run on
 * @fn:  the function to run
 * @arg: the function argument
 * @key: The lock class key for lock debugging purposes
 *
 * Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
 * any locks which would prevent @fn from completing.
 *
 * Return: The value @fn returns.
 */
long work_on_cpu_safe_key(int cpu, long (*fn)(void *),
                          void *arg, struct lock_class_key *key)
{
        long ret = -ENODEV;

        cpus_read_lock();
        if (cpu_online(cpu))
                ret = work_on_cpu_key(cpu, fn, arg, key);
        cpus_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(work_on_cpu_safe_key);
#endif /* CONFIG_SMP */

#ifdef CONFIG_FREEZER

/**
 * freeze_workqueues_begin - begin freezing workqueues
 *
 * Start freezing workqueues.  After this function returns, all freezable
 * workqueues will queue new works to their inactive_works list instead of
 * pool->worklist.
 *
 * CONTEXT:
 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
 */
void freeze_workqueues_begin(void)
{
        struct workqueue_struct *wq;

        mutex_lock(&wq_pool_mutex);

        WARN_ON_ONCE(workqueue_freezing);
        workqueue_freezing = true;

        list_for_each_entry(wq, &workqueues, list) {
                mutex_lock(&wq->mutex);
                wq_adjust_max_active(wq);
                mutex_unlock(&wq->mutex);
        }

        mutex_unlock(&wq_pool_mutex);
}

/**
 * freeze_workqueues_busy - are freezable workqueues still busy?
 *
 * Check whether freezing is complete.  This function must be called
 * between freeze_workqueues_begin() and thaw_workqueues().
 *
 * CONTEXT:
 * Grabs and releases wq_pool_mutex.
 *
 * Return:
 * %true if some freezable workqueues are still busy.  %false if freezing
 * is complete.
 */
bool freeze_workqueues_busy(void)
{
        bool busy = false;
        struct workqueue_struct *wq;
        struct pool_workqueue *pwq;

        mutex_lock(&wq_pool_mutex);

        WARN_ON_ONCE(!workqueue_freezing);

        list_for_each_entry(wq, &workqueues, list) {
                if (!(wq->flags & WQ_FREEZABLE))
                        continue;
                /*
                 * nr_active is monotonically decreasing.  It's safe
                 * to peek without lock.
                 */
                rcu_read_lock();
                for_each_pwq(pwq, wq) {
                        WARN_ON_ONCE(pwq->nr_active < 0);
                        if (pwq->nr_active) {
                                busy = true;
                                rcu_read_unlock();
                                goto out_unlock;
                        }
                }
                rcu_read_unlock();
        }
out_unlock:
        mutex_unlock(&wq_pool_mutex);
        return busy;
}

/**
 * thaw_workqueues - thaw workqueues
 *
 * Thaw workqueues.  Normal queueing is restored and all collected
 * frozen works are transferred to their respective pool worklists.
 *
 * CONTEXT:
 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
 */
void thaw_workqueues(void)
{
        struct workqueue_struct *wq;

        mutex_lock(&wq_pool_mutex);

        if (!workqueue_freezing)
                goto out_unlock;

        workqueue_freezing = false;

        /* restore max_active and repopulate worklist */
        list_for_each_entry(wq, &workqueues, list) {
                mutex_lock(&wq->mutex);
                wq_adjust_max_active(wq);
                mutex_unlock(&wq->mutex);
        }

out_unlock:
        mutex_unlock(&wq_pool_mutex);
}
#endif /* CONFIG_FREEZER */

static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)
{
        LIST_HEAD(ctxs);
        int ret = 0;
        struct workqueue_struct *wq;
        struct apply_wqattrs_ctx *ctx, *n;

        lockdep_assert_held(&wq_pool_mutex);

        list_for_each_entry(wq, &workqueues, list) {
                if (!(wq->flags & WQ_UNBOUND) || (wq->flags & __WQ_DESTROYING))
                        continue;

                ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask);
                if (IS_ERR(ctx)) {
                        ret = PTR_ERR(ctx);
                        break;
                }

                list_add_tail(&ctx->list, &ctxs);
        }

        list_for_each_entry_safe(ctx, n, &ctxs, list) {
                if (!ret)
                        apply_wqattrs_commit(ctx);
                apply_wqattrs_cleanup(ctx);
        }

        if (!ret) {
                mutex_lock(&wq_pool_attach_mutex);
                cpumask_copy(wq_unbound_cpumask, unbound_cpumask);
                mutex_unlock(&wq_pool_attach_mutex);
        }
        return ret;
}

/**
 * workqueue_unbound_exclude_cpumask - Exclude given CPUs from unbound cpumask
 * @exclude_cpumask: the cpumask to be excluded from wq_unbound_cpumask
 *
 * This function can be called from cpuset code to provide a set of isolated
 * CPUs that should be excluded from wq_unbound_cpumask.
 */
int workqueue_unbound_exclude_cpumask(cpumask_var_t exclude_cpumask)
{
        cpumask_var_t cpumask;
        int ret = 0;

        if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
                return -ENOMEM;

        mutex_lock(&wq_pool_mutex);

        /*
         * If the operation fails, it will fall back to
         * wq_requested_unbound_cpumask which is initially set to
         * (HK_TYPE_WQ ∩ HK_TYPE_DOMAIN) house keeping mask and rewritten
         * by any subsequent write to workqueue/cpumask sysfs file.
         */
        if (!cpumask_andnot(cpumask, wq_requested_unbound_cpumask, exclude_cpumask))
                cpumask_copy(cpumask, wq_requested_unbound_cpumask);
        if (!cpumask_equal(cpumask, wq_unbound_cpumask))
                ret = workqueue_apply_unbound_cpumask(cpumask);

        /* Save the current isolated cpumask & export it via sysfs */
        if (!ret)
                cpumask_copy(wq_isolated_cpumask, exclude_cpumask);

        mutex_unlock(&wq_pool_mutex);
        free_cpumask_var(cpumask);
        return ret;
}

static int parse_affn_scope(const char *val)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(wq_affn_names); i++) {
                if (!strncasecmp(val, wq_affn_names[i], strlen(wq_affn_names[i])))
                        return i;
        }
        return -EINVAL;
}

static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp)
{
        struct workqueue_struct *wq;
        int affn, cpu;

        affn = parse_affn_scope(val);
        if (affn < 0)
                return affn;
        if (affn == WQ_AFFN_DFL)
                return -EINVAL;

        cpus_read_lock();
        mutex_lock(&wq_pool_mutex);

        wq_affn_dfl = affn;

        list_for_each_entry(wq, &workqueues, list) {
                for_each_online_cpu(cpu)
                        unbound_wq_update_pwq(wq, cpu);
        }

        mutex_unlock(&wq_pool_mutex);
        cpus_read_unlock();

        return 0;
}

static int wq_affn_dfl_get(char *buffer, const struct kernel_param *kp)
{
        return scnprintf(buffer, PAGE_SIZE, "%s\n", wq_affn_names[wq_affn_dfl]);
}

static const struct kernel_param_ops wq_affn_dfl_ops = {
        .set        = wq_affn_dfl_set,
        .get        = wq_affn_dfl_get,
};

module_param_cb(default_affinity_scope, &wq_affn_dfl_ops, NULL, 0644);

#ifdef CONFIG_SYSFS
/*
 * Workqueues with WQ_SYSFS flag set is visible to userland via
 * /sys/bus/workqueue/devices/WQ_NAME.  All visible workqueues have the
 * following attributes.
 *
 *  per_cpu                RO bool        : whether the workqueue is per-cpu or unbound
 *  max_active                RW int        : maximum number of in-flight work items
 *
 * Unbound workqueues have the following extra attributes.
 *
 *  nice                RW int        : nice value of the workers
 *  cpumask                RW mask        : bitmask of allowed CPUs for the workers
 *  affinity_scope        RW str  : worker CPU affinity scope (cache, numa, none)
 *  affinity_strict        RW bool : worker CPU affinity is strict
 */
struct wq_device {
        struct workqueue_struct                *wq;
        struct device                        dev;
};

static struct workqueue_struct *dev_to_wq(struct device *dev)
{
        struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);

        return wq_dev->wq;
}

static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
                            char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);

        return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
}
static DEVICE_ATTR_RO(per_cpu);

static ssize_t max_active_show(struct device *dev,
                               struct device_attribute *attr, char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);

        return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
}

static ssize_t max_active_store(struct device *dev,
                                struct device_attribute *attr, const char *buf,
                                size_t count)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        int val;

        if (sscanf(buf, "%d", &val) != 1 || val <= 0)
                return -EINVAL;

        workqueue_set_max_active(wq, val);
        return count;
}
static DEVICE_ATTR_RW(max_active);

static struct attribute *wq_sysfs_attrs[] = {
        &dev_attr_per_cpu.attr,
        &dev_attr_max_active.attr,
        NULL,
};
ATTRIBUTE_GROUPS(wq_sysfs);

static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
                            char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        int written;

        mutex_lock(&wq->mutex);
        written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
        mutex_unlock(&wq->mutex);

        return written;
}

/* prepare workqueue_attrs for sysfs store operations */
static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
{
        struct workqueue_attrs *attrs;

        lockdep_assert_held(&wq_pool_mutex);

        attrs = alloc_workqueue_attrs();
        if (!attrs)
                return NULL;

        copy_workqueue_attrs(attrs, wq->unbound_attrs);
        return attrs;
}

static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
                             const char *buf, size_t count)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        struct workqueue_attrs *attrs;
        int ret = -ENOMEM;

        apply_wqattrs_lock();

        attrs = wq_sysfs_prep_attrs(wq);
        if (!attrs)
                goto out_unlock;

        if (sscanf(buf, "%d", &attrs->nice) == 1 &&
            attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
                ret = apply_workqueue_attrs_locked(wq, attrs);
        else
                ret = -EINVAL;

out_unlock:
        apply_wqattrs_unlock();
        free_workqueue_attrs(attrs);
        return ret ?: count;
}

static ssize_t wq_cpumask_show(struct device *dev,
                               struct device_attribute *attr, char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        int written;

        mutex_lock(&wq->mutex);
        written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
                            cpumask_pr_args(wq->unbound_attrs->cpumask));
        mutex_unlock(&wq->mutex);
        return written;
}

static ssize_t wq_cpumask_store(struct device *dev,
                                struct device_attribute *attr,
                                const char *buf, size_t count)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        struct workqueue_attrs *attrs;
        int ret = -ENOMEM;

        apply_wqattrs_lock();

        attrs = wq_sysfs_prep_attrs(wq);
        if (!attrs)
                goto out_unlock;

        ret = cpumask_parse(buf, attrs->cpumask);
        if (!ret)
                ret = apply_workqueue_attrs_locked(wq, attrs);

out_unlock:
        apply_wqattrs_unlock();
        free_workqueue_attrs(attrs);
        return ret ?: count;
}

static ssize_t wq_affn_scope_show(struct device *dev,
                                  struct device_attribute *attr, char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        int written;

        mutex_lock(&wq->mutex);
        if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL)
                written = scnprintf(buf, PAGE_SIZE, "%s (%s)\n",
                                    wq_affn_names[WQ_AFFN_DFL],
                                    wq_affn_names[wq_affn_dfl]);
        else
                written = scnprintf(buf, PAGE_SIZE, "%s\n",
                                    wq_affn_names[wq->unbound_attrs->affn_scope]);
        mutex_unlock(&wq->mutex);

        return written;
}

static ssize_t wq_affn_scope_store(struct device *dev,
                                   struct device_attribute *attr,
                                   const char *buf, size_t count)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        struct workqueue_attrs *attrs;
        int affn, ret = -ENOMEM;

        affn = parse_affn_scope(buf);
        if (affn < 0)
                return affn;

        apply_wqattrs_lock();
        attrs = wq_sysfs_prep_attrs(wq);
        if (attrs) {
                attrs->affn_scope = affn;
                ret = apply_workqueue_attrs_locked(wq, attrs);
        }
        apply_wqattrs_unlock();
        free_workqueue_attrs(attrs);
        return ret ?: count;
}

static ssize_t wq_affinity_strict_show(struct device *dev,
                                       struct device_attribute *attr, char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);

        return scnprintf(buf, PAGE_SIZE, "%d\n",
                         wq->unbound_attrs->affn_strict);
}

static ssize_t wq_affinity_strict_store(struct device *dev,
                                        struct device_attribute *attr,
                                        const char *buf, size_t count)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        struct workqueue_attrs *attrs;
        int v, ret = -ENOMEM;

        if (sscanf(buf, "%d", &v) != 1)
                return -EINVAL;

        apply_wqattrs_lock();
        attrs = wq_sysfs_prep_attrs(wq);
        if (attrs) {
                attrs->affn_strict = (bool)v;
                ret = apply_workqueue_attrs_locked(wq, attrs);
        }
        apply_wqattrs_unlock();
        free_workqueue_attrs(attrs);
        return ret ?: count;
}

static struct device_attribute wq_sysfs_unbound_attrs[] = {
        __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
        __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
        __ATTR(affinity_scope, 0644, wq_affn_scope_show, wq_affn_scope_store),
        __ATTR(affinity_strict, 0644, wq_affinity_strict_show, wq_affinity_strict_store),
        __ATTR_NULL,
};

static const struct bus_type wq_subsys = {
        .name                                = "workqueue",
        .dev_groups                        = wq_sysfs_groups,
};

/**
 *  workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
 *  @cpumask: the cpumask to set
 *
 *  The low-level workqueues cpumask is a global cpumask that limits
 *  the affinity of all unbound workqueues.  This function check the @cpumask
 *  and apply it to all unbound workqueues and updates all pwqs of them.
 *
 *  Return:        0        - Success
 *                -EINVAL        - Invalid @cpumask
 *                -ENOMEM        - Failed to allocate memory for attrs or pwqs.
 */
static int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
{
        int ret = -EINVAL;

        /*
         * Not excluding isolated cpus on purpose.
         * If the user wishes to include them, we allow that.
         */
        cpumask_and(cpumask, cpumask, cpu_possible_mask);
        if (!cpumask_empty(cpumask)) {
                ret = 0;
                apply_wqattrs_lock();
                if (!cpumask_equal(cpumask, wq_unbound_cpumask))
                        ret = workqueue_apply_unbound_cpumask(cpumask);
                if (!ret)
                        cpumask_copy(wq_requested_unbound_cpumask, cpumask);
                apply_wqattrs_unlock();
        }

        return ret;
}

static ssize_t __wq_cpumask_show(struct device *dev,
                struct device_attribute *attr, char *buf, cpumask_var_t mask)
{
        int written;

        mutex_lock(&wq_pool_mutex);
        written = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(mask));
        mutex_unlock(&wq_pool_mutex);

        return written;
}

static ssize_t cpumask_requested_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask);
}
static DEVICE_ATTR_RO(cpumask_requested);

static ssize_t cpumask_isolated_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask);
}
static DEVICE_ATTR_RO(cpumask_isolated);

static ssize_t cpumask_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask);
}

static ssize_t cpumask_store(struct device *dev,
                struct device_attribute *attr, const char *buf, size_t count)
{
        cpumask_var_t cpumask;
        int ret;

        if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
                return -ENOMEM;

        ret = cpumask_parse(buf, cpumask);
        if (!ret)
                ret = workqueue_set_unbound_cpumask(cpumask);

        free_cpumask_var(cpumask);
        return ret ? ret : count;
}
static DEVICE_ATTR_RW(cpumask);

static struct attribute *wq_sysfs_cpumask_attrs[] = {
        &dev_attr_cpumask.attr,
        &dev_attr_cpumask_requested.attr,
        &dev_attr_cpumask_isolated.attr,
        NULL,
};
ATTRIBUTE_GROUPS(wq_sysfs_cpumask);

static int __init wq_sysfs_init(void)
{
        return subsys_virtual_register(&wq_subsys, wq_sysfs_cpumask_groups);
}
core_initcall(wq_sysfs_init);

static void wq_device_release(struct device *dev)
{
        struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);

        kfree(wq_dev);
}

/**
 * workqueue_sysfs_register - make a workqueue visible in sysfs
 * @wq: the workqueue to register
 *
 * Expose @wq in sysfs under /sys/bus/workqueue/devices.
 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
 * which is the preferred method.
 *
 * Workqueue user should use this function directly iff it wants to apply
 * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
 * apply_workqueue_attrs() may race against userland updating the
 * attributes.
 *
 * Return: 0 on success, -errno on failure.
 */
int workqueue_sysfs_register(struct workqueue_struct *wq)
{
        struct wq_device *wq_dev;
        int ret;

        /*
         * Adjusting max_active breaks ordering guarantee.  Disallow exposing
         * ordered workqueues.
         */
        if (WARN_ON(wq->flags & __WQ_ORDERED))
                return -EINVAL;

        wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
        if (!wq_dev)
                return -ENOMEM;

        wq_dev->wq = wq;
        wq_dev->dev.bus = &wq_subsys;
        wq_dev->dev.release = wq_device_release;
        dev_set_name(&wq_dev->dev, "%s", wq->name);

        /*
         * unbound_attrs are created separately.  Suppress uevent until
         * everything is ready.
         */
        dev_set_uevent_suppress(&wq_dev->dev, true);

        ret = device_register(&wq_dev->dev);
        if (ret) {
                put_device(&wq_dev->dev);
                wq->wq_dev = NULL;
                return ret;
        }

        if (wq->flags & WQ_UNBOUND) {
                struct device_attribute *attr;

                for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
                        ret = device_create_file(&wq_dev->dev, attr);
                        if (ret) {
                                device_unregister(&wq_dev->dev);
                                wq->wq_dev = NULL;
                                return ret;
                        }
                }
        }

        dev_set_uevent_suppress(&wq_dev->dev, false);
        kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
        return 0;
}

/**
 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
 * @wq: the workqueue to unregister
 *
 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
 */
static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
{
        struct wq_device *wq_dev = wq->wq_dev;

        if (!wq->wq_dev)
                return;

        wq->wq_dev = NULL;
        device_unregister(&wq_dev->dev);
}
#else        /* CONFIG_SYSFS */
static void workqueue_sysfs_unregister(struct workqueue_struct *wq)        { }
#endif        /* CONFIG_SYSFS */

/*
 * Workqueue watchdog.
 *
 * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
 * flush dependency, a concurrency managed work item which stays RUNNING
 * indefinitely.  Workqueue stalls can be very difficult to debug as the
 * usual warning mechanisms don't trigger and internal workqueue state is
 * largely opaque.
 *
 * Workqueue watchdog monitors all worker pools periodically and dumps
 * state if some pools failed to make forward progress for a while where
 * forward progress is defined as the first item on ->worklist changing.
 *
 * This mechanism is controlled through the kernel parameter
 * "workqueue.watchdog_thresh" which can be updated at runtime through the
 * corresponding sysfs parameter file.
 */
#ifdef CONFIG_WQ_WATCHDOG

static unsigned long wq_watchdog_thresh = 30;
static struct timer_list wq_watchdog_timer;

static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;

static unsigned int wq_panic_on_stall;
module_param_named(panic_on_stall, wq_panic_on_stall, uint, 0644);

/*
 * Show workers that might prevent the processing of pending work items.
 * The only candidates are CPU-bound workers in the running state.
 * Pending work items should be handled by another idle worker
 * in all other situations.
 */
static void show_cpu_pool_hog(struct worker_pool *pool)
{
        struct worker *worker;
        unsigned long irq_flags;
        int bkt;

        raw_spin_lock_irqsave(&pool->lock, irq_flags);

        hash_for_each(pool->busy_hash, bkt, worker, hentry) {
                if (task_is_running(worker->task)) {
                        /*
                         * Defer printing to avoid deadlocks in console
                         * drivers that queue work while holding locks
                         * also taken in their write paths.
                         */
                        printk_deferred_enter();

                        pr_info("pool %d:\n", pool->id);
                        sched_show_task(worker->task);

                        printk_deferred_exit();
                }
        }

        raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
}

static void show_cpu_pools_hogs(void)
{
        struct worker_pool *pool;
        int pi;

        pr_info("Showing backtraces of running workers in stalled CPU-bound worker pools:\n");

        rcu_read_lock();

        for_each_pool(pool, pi) {
                if (pool->cpu_stall)
                        show_cpu_pool_hog(pool);

        }

        rcu_read_unlock();
}

static void panic_on_wq_watchdog(void)
{
        static unsigned int wq_stall;

        if (wq_panic_on_stall) {
                wq_stall++;
                BUG_ON(wq_stall >= wq_panic_on_stall);
        }
}

static void wq_watchdog_reset_touched(void)
{
        int cpu;

        wq_watchdog_touched = jiffies;
        for_each_possible_cpu(cpu)
                per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
}

static void wq_watchdog_timer_fn(struct timer_list *unused)
{
        unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
        bool lockup_detected = false;
        bool cpu_pool_stall = false;
        unsigned long now = jiffies;
        struct worker_pool *pool;
        int pi;

        if (!thresh)
                return;

        rcu_read_lock();

        for_each_pool(pool, pi) {
                unsigned long pool_ts, touched, ts;

                pool->cpu_stall = false;
                if (list_empty(&pool->worklist))
                        continue;

                /*
                 * If a virtual machine is stopped by the host it can look to
                 * the watchdog like a stall.
                 */
                kvm_check_and_clear_guest_paused();

                /* get the latest of pool and touched timestamps */
                if (pool->cpu >= 0)
                        touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
                else
                        touched = READ_ONCE(wq_watchdog_touched);
                pool_ts = READ_ONCE(pool->watchdog_ts);

                if (time_after(pool_ts, touched))
                        ts = pool_ts;
                else
                        ts = touched;

                /* did we stall? */
                if (time_after(now, ts + thresh)) {
                        lockup_detected = true;
                        if (pool->cpu >= 0 && !(pool->flags & POOL_BH)) {
                                pool->cpu_stall = true;
                                cpu_pool_stall = true;
                        }
                        pr_emerg("BUG: workqueue lockup - pool");
                        pr_cont_pool_info(pool);
                        pr_cont(" stuck for %us!\n",
                                jiffies_to_msecs(now - pool_ts) / 1000);
                }


        }

        rcu_read_unlock();

        if (lockup_detected)
                show_all_workqueues();

        if (cpu_pool_stall)
                show_cpu_pools_hogs();

        if (lockup_detected)
                panic_on_wq_watchdog();

        wq_watchdog_reset_touched();
        mod_timer(&wq_watchdog_timer, jiffies + thresh);
}

notrace void wq_watchdog_touch(int cpu)
{
        unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
        unsigned long touch_ts = READ_ONCE(wq_watchdog_touched);
        unsigned long now = jiffies;

        if (cpu >= 0)
                per_cpu(wq_watchdog_touched_cpu, cpu) = now;
        else
                WARN_ONCE(1, "%s should be called with valid CPU", __func__);

        /* Don't unnecessarily store to global cacheline */
        if (time_after(now, touch_ts + thresh / 4))
                WRITE_ONCE(wq_watchdog_touched, jiffies);
}

static void wq_watchdog_set_thresh(unsigned long thresh)
{
        wq_watchdog_thresh = 0;
        del_timer_sync(&wq_watchdog_timer);

        if (thresh) {
                wq_watchdog_thresh = thresh;
                wq_watchdog_reset_touched();
                mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
        }
}

static int wq_watchdog_param_set_thresh(const char *val,
                                        const struct kernel_param *kp)
{
        unsigned long thresh;
        int ret;

        ret = kstrtoul(val, 0, &thresh);
        if (ret)
                return ret;

        if (system_wq)
                wq_watchdog_set_thresh(thresh);
        else
                wq_watchdog_thresh = thresh;

        return 0;
}

static const struct kernel_param_ops wq_watchdog_thresh_ops = {
        .set        = wq_watchdog_param_set_thresh,
        .get        = param_get_ulong,
};

module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
                0644);

static void wq_watchdog_init(void)
{
        timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
        wq_watchdog_set_thresh(wq_watchdog_thresh);
}

#else        /* CONFIG_WQ_WATCHDOG */

static inline void wq_watchdog_init(void) { }

#endif        /* CONFIG_WQ_WATCHDOG */

static void bh_pool_kick_normal(struct irq_work *irq_work)
{
        raise_softirq_irqoff(TASKLET_SOFTIRQ);
}

static void bh_pool_kick_highpri(struct irq_work *irq_work)
{
        raise_softirq_irqoff(HI_SOFTIRQ);
}

static void __init restrict_unbound_cpumask(const char *name, const struct cpumask *mask)
{
        if (!cpumask_intersects(wq_unbound_cpumask, mask)) {
                pr_warn("workqueue: Restricting unbound_cpumask (%*pb) with %s (%*pb) leaves no CPU, ignoring\n",
                        cpumask_pr_args(wq_unbound_cpumask), name, cpumask_pr_args(mask));
                return;
        }

        cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, mask);
}

static void __init init_cpu_worker_pool(struct worker_pool *pool, int cpu, int nice)
{
        BUG_ON(init_worker_pool(pool));
        pool->cpu = cpu;
        cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
        cpumask_copy(pool->attrs->__pod_cpumask, cpumask_of(cpu));
        pool->attrs->nice = nice;
        pool->attrs->affn_strict = true;
        pool->node = cpu_to_node(cpu);

        /* alloc pool ID */
        mutex_lock(&wq_pool_mutex);
        BUG_ON(worker_pool_assign_id(pool));
        mutex_unlock(&wq_pool_mutex);
}

/**
 * workqueue_init_early - early init for workqueue subsystem
 *
 * This is the first step of three-staged workqueue subsystem initialization and
 * invoked as soon as the bare basics - memory allocation, cpumasks and idr are
 * up. It sets up all the data structures and system workqueues and allows early
 * boot code to create workqueues and queue/cancel work items. Actual work item
 * execution starts only after kthreads can be created and scheduled right
 * before early initcalls.
 */
void __init workqueue_init_early(void)
{
        struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM];
        int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
        void (*irq_work_fns[2])(struct irq_work *) = { bh_pool_kick_normal,
                                                       bh_pool_kick_highpri };
        int i, cpu;

        BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));

        BUG_ON(!alloc_cpumask_var(&wq_online_cpumask, GFP_KERNEL));
        BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
        BUG_ON(!alloc_cpumask_var(&wq_requested_unbound_cpumask, GFP_KERNEL));
        BUG_ON(!zalloc_cpumask_var(&wq_isolated_cpumask, GFP_KERNEL));

        cpumask_copy(wq_online_cpumask, cpu_online_mask);
        cpumask_copy(wq_unbound_cpumask, cpu_possible_mask);
        restrict_unbound_cpumask("HK_TYPE_WQ", housekeeping_cpumask(HK_TYPE_WQ));
        restrict_unbound_cpumask("HK_TYPE_DOMAIN", housekeeping_cpumask(HK_TYPE_DOMAIN));
        if (!cpumask_empty(&wq_cmdline_cpumask))
                restrict_unbound_cpumask("workqueue.unbound_cpus", &wq_cmdline_cpumask);

        cpumask_copy(wq_requested_unbound_cpumask, wq_unbound_cpumask);

        pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);

        unbound_wq_update_pwq_attrs_buf = alloc_workqueue_attrs();
        BUG_ON(!unbound_wq_update_pwq_attrs_buf);

        /*
         * If nohz_full is enabled, set power efficient workqueue as unbound.
         * This allows workqueue items to be moved to HK CPUs.
         */
        if (housekeeping_enabled(HK_TYPE_TICK))
                wq_power_efficient = true;

        /* initialize WQ_AFFN_SYSTEM pods */
        pt->pod_cpus = kcalloc(1, sizeof(pt->pod_cpus[0]), GFP_KERNEL);
        pt->pod_node = kcalloc(1, sizeof(pt->pod_node[0]), GFP_KERNEL);
        pt->cpu_pod = kcalloc(nr_cpu_ids, sizeof(pt->cpu_pod[0]), GFP_KERNEL);
        BUG_ON(!pt->pod_cpus || !pt->pod_node || !pt->cpu_pod);

        BUG_ON(!zalloc_cpumask_var_node(&pt->pod_cpus[0], GFP_KERNEL, NUMA_NO_NODE));

        pt->nr_pods = 1;
        cpumask_copy(pt->pod_cpus[0], cpu_possible_mask);
        pt->pod_node[0] = NUMA_NO_NODE;
        pt->cpu_pod[0] = 0;

        /* initialize BH and CPU pools */
        for_each_possible_cpu(cpu) {
                struct worker_pool *pool;

                i = 0;
                for_each_bh_worker_pool(pool, cpu) {
                        init_cpu_worker_pool(pool, cpu, std_nice[i]);
                        pool->flags |= POOL_BH;
                        init_irq_work(bh_pool_irq_work(pool), irq_work_fns[i]);
                        i++;
                }

                i = 0;
                for_each_cpu_worker_pool(pool, cpu)
                        init_cpu_worker_pool(pool, cpu, std_nice[i++]);
        }

        /* create default unbound and ordered wq attrs */
        for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
                struct workqueue_attrs *attrs;

                BUG_ON(!(attrs = alloc_workqueue_attrs()));
                attrs->nice = std_nice[i];
                unbound_std_wq_attrs[i] = attrs;

                /*
                 * An ordered wq should have only one pwq as ordering is
                 * guaranteed by max_active which is enforced by pwqs.
                 */
                BUG_ON(!(attrs = alloc_workqueue_attrs()));
                attrs->nice = std_nice[i];
                attrs->ordered = true;
                ordered_wq_attrs[i] = attrs;
        }

        system_wq = alloc_workqueue("events", 0, 0);
        system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
        system_long_wq = alloc_workqueue("events_long", 0, 0);
        system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
                                            WQ_MAX_ACTIVE);
        system_freezable_wq = alloc_workqueue("events_freezable",
                                              WQ_FREEZABLE, 0);
        system_power_efficient_wq = alloc_workqueue("events_power_efficient",
                                              WQ_POWER_EFFICIENT, 0);
        system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_pwr_efficient",
                                              WQ_FREEZABLE | WQ_POWER_EFFICIENT,
                                              0);
        system_bh_wq = alloc_workqueue("events_bh", WQ_BH, 0);
        system_bh_highpri_wq = alloc_workqueue("events_bh_highpri",
                                               WQ_BH | WQ_HIGHPRI, 0);
        BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
               !system_unbound_wq || !system_freezable_wq ||
               !system_power_efficient_wq ||
               !system_freezable_power_efficient_wq ||
               !system_bh_wq || !system_bh_highpri_wq);
}

static void __init wq_cpu_intensive_thresh_init(void)
{
        unsigned long thresh;
        unsigned long bogo;

        pwq_release_worker = kthread_create_worker(0, "pool_workqueue_release");
        BUG_ON(IS_ERR(pwq_release_worker));

        /* if the user set it to a specific value, keep it */
        if (wq_cpu_intensive_thresh_us != ULONG_MAX)
                return;

        /*
         * The default of 10ms is derived from the fact that most modern (as of
         * 2023) processors can do a lot in 10ms and that it's just below what
         * most consider human-perceivable. However, the kernel also runs on a
         * lot slower CPUs including microcontrollers where the threshold is way
         * too low.
         *
         * Let's scale up the threshold upto 1 second if BogoMips is below 4000.
         * This is by no means accurate but it doesn't have to be. The mechanism
         * is still useful even when the threshold is fully scaled up. Also, as
         * the reports would usually be applicable to everyone, some machines
         * operating on longer thresholds won't significantly diminish their
         * usefulness.
         */
        thresh = 10 * USEC_PER_MSEC;

        /* see init/calibrate.c for lpj -> BogoMIPS calculation */
        bogo = max_t(unsigned long, loops_per_jiffy / 500000 * HZ, 1);
        if (bogo < 4000)
                thresh = min_t(unsigned long, thresh * 4000 / bogo, USEC_PER_SEC);

        pr_debug("wq_cpu_intensive_thresh: lpj=%lu BogoMIPS=%lu thresh_us=%lu\n",
                 loops_per_jiffy, bogo, thresh);

        wq_cpu_intensive_thresh_us = thresh;
}

/**
 * workqueue_init - bring workqueue subsystem fully online
 *
 * This is the second step of three-staged workqueue subsystem initialization
 * and invoked as soon as kthreads can be created and scheduled. Workqueues have
 * been created and work items queued on them, but there are no kworkers
 * executing the work items yet. Populate the worker pools with the initial
 * workers and enable future kworker creations.
 */
void __init workqueue_init(void)
{
        struct workqueue_struct *wq;
        struct worker_pool *pool;
        int cpu, bkt;

        wq_cpu_intensive_thresh_init();

        mutex_lock(&wq_pool_mutex);

        /*
         * Per-cpu pools created earlier could be missing node hint. Fix them
         * up. Also, create a rescuer for workqueues that requested it.
         */
        for_each_possible_cpu(cpu) {
                for_each_bh_worker_pool(pool, cpu)
                        pool->node = cpu_to_node(cpu);
                for_each_cpu_worker_pool(pool, cpu)
                        pool->node = cpu_to_node(cpu);
        }

        list_for_each_entry(wq, &workqueues, list) {
                WARN(init_rescuer(wq),
                     "workqueue: failed to create early rescuer for %s",
                     wq->name);
        }

        mutex_unlock(&wq_pool_mutex);

        /*
         * Create the initial workers. A BH pool has one pseudo worker that
         * represents the shared BH execution context and thus doesn't get
         * affected by hotplug events. Create the BH pseudo workers for all
         * possible CPUs here.
         */
        for_each_possible_cpu(cpu)
                for_each_bh_worker_pool(pool, cpu)
                        BUG_ON(!create_worker(pool));

        for_each_online_cpu(cpu) {
                for_each_cpu_worker_pool(pool, cpu) {
                        pool->flags &= ~POOL_DISASSOCIATED;
                        BUG_ON(!create_worker(pool));
                }
        }

        hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
                BUG_ON(!create_worker(pool));

        wq_online = true;
        wq_watchdog_init();
}

/*
 * Initialize @pt by first initializing @pt->cpu_pod[] with pod IDs according to
 * @cpu_shares_pod(). Each subset of CPUs that share a pod is assigned a unique
 * and consecutive pod ID. The rest of @pt is initialized accordingly.
 */
static void __init init_pod_type(struct wq_pod_type *pt,
                                 bool (*cpus_share_pod)(int, int))
{
        int cur, pre, cpu, pod;

        pt->nr_pods = 0;

        /* init @pt->cpu_pod[] according to @cpus_share_pod() */
        pt->cpu_pod = kcalloc(nr_cpu_ids, sizeof(pt->cpu_pod[0]), GFP_KERNEL);
        BUG_ON(!pt->cpu_pod);

        for_each_possible_cpu(cur) {
                for_each_possible_cpu(pre) {
                        if (pre >= cur) {
                                pt->cpu_pod[cur] = pt->nr_pods++;
                                break;
                        }
                        if (cpus_share_pod(cur, pre)) {
                                pt->cpu_pod[cur] = pt->cpu_pod[pre];
                                break;
                        }
                }
        }

        /* init the rest to match @pt->cpu_pod[] */
        pt->pod_cpus = kcalloc(pt->nr_pods, sizeof(pt->pod_cpus[0]), GFP_KERNEL);
        pt->pod_node = kcalloc(pt->nr_pods, sizeof(pt->pod_node[0]), GFP_KERNEL);
        BUG_ON(!pt->pod_cpus || !pt->pod_node);

        for (pod = 0; pod < pt->nr_pods; pod++)
                BUG_ON(!zalloc_cpumask_var(&pt->pod_cpus[pod], GFP_KERNEL));

        for_each_possible_cpu(cpu) {
                cpumask_set_cpu(cpu, pt->pod_cpus[pt->cpu_pod[cpu]]);
                pt->pod_node[pt->cpu_pod[cpu]] = cpu_to_node(cpu);
        }
}

static bool __init cpus_dont_share(int cpu0, int cpu1)
{
        return false;
}

static bool __init cpus_share_smt(int cpu0, int cpu1)
{
#ifdef CONFIG_SCHED_SMT
        return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));
#else
        return false;
#endif
}

static bool __init cpus_share_numa(int cpu0, int cpu1)
{
        return cpu_to_node(cpu0) == cpu_to_node(cpu1);
}

/**
 * workqueue_init_topology - initialize CPU pods for unbound workqueues
 *
 * This is the third step of three-staged workqueue subsystem initialization and
 * invoked after SMP and topology information are fully initialized. It
 * initializes the unbound CPU pods accordingly.
 */
void __init workqueue_init_topology(void)
{
        struct workqueue_struct *wq;
        int cpu;

        init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share);
        init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt);
        init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache);
        init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa);

        wq_topo_initialized = true;

        mutex_lock(&wq_pool_mutex);

        /*
         * Workqueues allocated earlier would have all CPUs sharing the default
         * worker pool. Explicitly call unbound_wq_update_pwq() on all workqueue
         * and CPU combinations to apply per-pod sharing.
         */
        list_for_each_entry(wq, &workqueues, list) {
                for_each_online_cpu(cpu)
                        unbound_wq_update_pwq(wq, cpu);
                if (wq->flags & WQ_UNBOUND) {
                        mutex_lock(&wq->mutex);
                        wq_update_node_max_active(wq, -1);
                        mutex_unlock(&wq->mutex);
                }
        }

        mutex_unlock(&wq_pool_mutex);
}

void __warn_flushing_systemwide_wq(void)
{
        pr_warn("WARNING: Flushing system-wide workqueues will be prohibited in near future.\n");
        dump_stack();
}
EXPORT_SYMBOL(__warn_flushing_systemwide_wq);

static int __init workqueue_unbound_cpus_setup(char *str)
{
        if (cpulist_parse(str, &wq_cmdline_cpumask) < 0) {
                cpumask_clear(&wq_cmdline_cpumask);
                pr_warn("workqueue.unbound_cpus: incorrect CPU range, using default\n");
        }

        return 1;
}
__setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup);
























































 7199 































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * AppArmor security module
 *
 * This file contains AppArmor network mediation definitions.
 *
 * Copyright (C) 1998-2008 Novell/SUSE
 * Copyright 2009-2017 Canonical Ltd.
 */

#ifndef __AA_NET_H
#define __AA_NET_H

#include <net/sock.h>
#include <linux/path.h>

#include "apparmorfs.h"
#include "label.h"
#include "perms.h"
#include "policy.h"

#define AA_MAY_SEND                AA_MAY_WRITE
#define AA_MAY_RECEIVE                AA_MAY_READ

#define AA_MAY_SHUTDOWN                AA_MAY_DELETE

#define AA_MAY_CONNECT                AA_MAY_OPEN
#define AA_MAY_ACCEPT                0x00100000

#define AA_MAY_BIND                0x00200000
#define AA_MAY_LISTEN                0x00400000

#define AA_MAY_SETOPT                0x01000000
#define AA_MAY_GETOPT                0x02000000

#define NET_PERMS_MASK (AA_MAY_SEND | AA_MAY_RECEIVE | AA_MAY_CREATE |    \
                        AA_MAY_SHUTDOWN | AA_MAY_BIND | AA_MAY_LISTEN |          \
                        AA_MAY_CONNECT | AA_MAY_ACCEPT | AA_MAY_SETATTR | \
                        AA_MAY_GETATTR | AA_MAY_SETOPT | AA_MAY_GETOPT)

#define NET_FS_PERMS (AA_MAY_SEND | AA_MAY_RECEIVE | AA_MAY_CREATE |        \
                      AA_MAY_SHUTDOWN | AA_MAY_CONNECT | AA_MAY_RENAME |\
                      AA_MAY_SETATTR | AA_MAY_GETATTR | AA_MAY_CHMOD |        \
                      AA_MAY_CHOWN | AA_MAY_CHGRP | AA_MAY_LOCK |        \
                      AA_MAY_MPROT)

#define NET_PEER_MASK (AA_MAY_SEND | AA_MAY_RECEIVE | AA_MAY_CONNECT |        \
                       AA_MAY_ACCEPT)
struct aa_sk_ctx {
        struct aa_label *label;
        struct aa_label *peer;
};

static inline struct aa_sk_ctx *aa_sock(const struct sock *sk)
{
        return sk->sk_security + apparmor_blob_sizes.lbs_sock;
}

#define DEFINE_AUDIT_NET(NAME, OP, SK, F, T, P)                                  \
        struct lsm_network_audit NAME ## _net = { .sk = (SK),                  \
                                                  .family = (F)};          \
        DEFINE_AUDIT_DATA(NAME,                                                  \
                          ((SK) && (F) != AF_UNIX) ? LSM_AUDIT_DATA_NET : \
                                                     LSM_AUDIT_DATA_NONE, \
                                                     AA_CLASS_NET,        \
                          OP);                                                  \
        NAME.common.u.net = &(NAME ## _net);                                  \
        NAME.net.type = (T);                                                  \
        NAME.net.protocol = (P)

#define DEFINE_AUDIT_SK(NAME, OP, SK)                                        \
        DEFINE_AUDIT_NET(NAME, OP, SK, (SK)->sk_family, (SK)->sk_type,        \
                         (SK)->sk_protocol)


#define af_select(FAMILY, FN, DEF_FN)                \
({                                                \
        int __e;                                \
        switch ((FAMILY)) {                        \
        default:                                \
                __e = DEF_FN;                        \
        }                                        \
        __e;                                        \
})

struct aa_secmark {
        u8 audit;
        u8 deny;
        u32 secid;
        char *label;
};

extern struct aa_sfs_entry aa_sfs_entry_network[];

void audit_net_cb(struct audit_buffer *ab, void *va);
int aa_profile_af_perm(struct aa_profile *profile,
                       struct apparmor_audit_data *ad,
                       u32 request, u16 family, int type);
int aa_af_perm(const struct cred *subj_cred, struct aa_label *label,
               const char *op, u32 request, u16 family,
               int type, int protocol);
static inline int aa_profile_af_sk_perm(struct aa_profile *profile,
                                        struct apparmor_audit_data *ad,
                                        u32 request,
                                        struct sock *sk)
{
        return aa_profile_af_perm(profile, ad, request, sk->sk_family,
                                  sk->sk_type);
}
int aa_sk_perm(const char *op, u32 request, struct sock *sk);

int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label,
                      const char *op, u32 request,
                      struct socket *sock);

int apparmor_secmark_check(struct aa_label *label, char *op, u32 request,
                           u32 secid, const struct sock *sk);

#endif /* __AA_NET_H */





















































































































































































































































































































































































































































































































































    3 




    3 
    3 
    3 
















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
// SPDX-License-Identifier: GPL-2.0
/*
 * This contains functions for filename crypto management
 *
 * Copyright (C) 2015, Google, Inc.
 * Copyright (C) 2015, Motorola Mobility
 *
 * Written by Uday Savagaonkar, 2014.
 * Modified by Jaegeuk Kim, 2015.
 *
 * This has not yet undergone a rigorous security audit.
 */

#include <linux/namei.h>
#include <linux/scatterlist.h>
#include <crypto/hash.h>
#include <crypto/sha2.h>
#include <crypto/skcipher.h>
#include "fscrypt_private.h"

/*
 * The minimum message length (input and output length), in bytes, for all
 * filenames encryption modes.  Filenames shorter than this will be zero-padded
 * before being encrypted.
 */
#define FSCRYPT_FNAME_MIN_MSG_LEN 16

/*
 * struct fscrypt_nokey_name - identifier for directory entry when key is absent
 *
 * When userspace lists an encrypted directory without access to the key, the
 * filesystem must present a unique "no-key name" for each filename that allows
 * it to find the directory entry again if requested.  Naively, that would just
 * mean using the ciphertext filenames.  However, since the ciphertext filenames
 * can contain illegal characters ('\0' and '/'), they must be encoded in some
 * way.  We use base64url.  But that can cause names to exceed NAME_MAX (255
 * bytes), so we also need to use a strong hash to abbreviate long names.
 *
 * The filesystem may also need another kind of hash, the "dirhash", to quickly
 * find the directory entry.  Since filesystems normally compute the dirhash
 * over the on-disk filename (i.e. the ciphertext), it's not computable from
 * no-key names that abbreviate the ciphertext using the strong hash to fit in
 * NAME_MAX.  It's also not computable if it's a keyed hash taken over the
 * plaintext (but it may still be available in the on-disk directory entry);
 * casefolded directories use this type of dirhash.  At least in these cases,
 * each no-key name must include the name's dirhash too.
 *
 * To meet all these requirements, we base64url-encode the following
 * variable-length structure.  It contains the dirhash, or 0's if the filesystem
 * didn't provide one; up to 149 bytes of the ciphertext name; and for
 * ciphertexts longer than 149 bytes, also the SHA-256 of the remaining bytes.
 *
 * This ensures that each no-key name contains everything needed to find the
 * directory entry again, contains only legal characters, doesn't exceed
 * NAME_MAX, is unambiguous unless there's a SHA-256 collision, and that we only
 * take the performance hit of SHA-256 on very long filenames (which are rare).
 */
struct fscrypt_nokey_name {
        u32 dirhash[2];
        u8 bytes[149];
        u8 sha256[SHA256_DIGEST_SIZE];
}; /* 189 bytes => 252 bytes base64url-encoded, which is <= NAME_MAX (255) */

/*
 * Decoded size of max-size no-key name, i.e. a name that was abbreviated using
 * the strong hash and thus includes the 'sha256' field.  This isn't simply
 * sizeof(struct fscrypt_nokey_name), as the padding at the end isn't included.
 */
#define FSCRYPT_NOKEY_NAME_MAX        offsetofend(struct fscrypt_nokey_name, sha256)

/* Encoded size of max-size no-key name */
#define FSCRYPT_NOKEY_NAME_MAX_ENCODED \
                FSCRYPT_BASE64URL_CHARS(FSCRYPT_NOKEY_NAME_MAX)

static inline bool fscrypt_is_dot_dotdot(const struct qstr *str)
{
        return is_dot_dotdot(str->name, str->len);
}

/**
 * fscrypt_fname_encrypt() - encrypt a filename
 * @inode: inode of the parent directory (for regular filenames)
 *           or of the symlink (for symlink targets). Key must already be
 *           set up.
 * @iname: the filename to encrypt
 * @out: (output) the encrypted filename
 * @olen: size of the encrypted filename.  It must be at least @iname->len.
 *          Any extra space is filled with NUL padding before encryption.
 *
 * Return: 0 on success, -errno on failure
 */
int fscrypt_fname_encrypt(const struct inode *inode, const struct qstr *iname,
                          u8 *out, unsigned int olen)
{
        struct skcipher_request *req = NULL;
        DECLARE_CRYPTO_WAIT(wait);
        const struct fscrypt_inode_info *ci = inode->i_crypt_info;
        struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;
        union fscrypt_iv iv;
        struct scatterlist sg;
        int res;

        /*
         * Copy the filename to the output buffer for encrypting in-place and
         * pad it with the needed number of NUL bytes.
         */
        if (WARN_ON_ONCE(olen < iname->len))
                return -ENOBUFS;
        memcpy(out, iname->name, iname->len);
        memset(out + iname->len, 0, olen - iname->len);

        /* Initialize the IV */
        fscrypt_generate_iv(&iv, 0, ci);

        /* Set up the encryption request */
        req = skcipher_request_alloc(tfm, GFP_NOFS);
        if (!req)
                return -ENOMEM;
        skcipher_request_set_callback(req,
                        CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
                        crypto_req_done, &wait);
        sg_init_one(&sg, out, olen);
        skcipher_request_set_crypt(req, &sg, &sg, olen, &iv);

        /* Do the encryption */
        res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
        skcipher_request_free(req);
        if (res < 0) {
                fscrypt_err(inode, "Filename encryption failed: %d", res);
                return res;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(fscrypt_fname_encrypt);

/**
 * fname_decrypt() - decrypt a filename
 * @inode: inode of the parent directory (for regular filenames)
 *           or of the symlink (for symlink targets)
 * @iname: the encrypted filename to decrypt
 * @oname: (output) the decrypted filename.  The caller must have allocated
 *           enough space for this, e.g. using fscrypt_fname_alloc_buffer().
 *
 * Return: 0 on success, -errno on failure
 */
static int fname_decrypt(const struct inode *inode,
                         const struct fscrypt_str *iname,
                         struct fscrypt_str *oname)
{
        struct skcipher_request *req = NULL;
        DECLARE_CRYPTO_WAIT(wait);
        struct scatterlist src_sg, dst_sg;
        const struct fscrypt_inode_info *ci = inode->i_crypt_info;
        struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;
        union fscrypt_iv iv;
        int res;

        /* Allocate request */
        req = skcipher_request_alloc(tfm, GFP_NOFS);
        if (!req)
                return -ENOMEM;
        skcipher_request_set_callback(req,
                CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
                crypto_req_done, &wait);

        /* Initialize IV */
        fscrypt_generate_iv(&iv, 0, ci);

        /* Create decryption request */
        sg_init_one(&src_sg, iname->name, iname->len);
        sg_init_one(&dst_sg, oname->name, oname->len);
        skcipher_request_set_crypt(req, &src_sg, &dst_sg, iname->len, &iv);
        res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait);
        skcipher_request_free(req);
        if (res < 0) {
                fscrypt_err(inode, "Filename decryption failed: %d", res);
                return res;
        }

        oname->len = strnlen(oname->name, iname->len);
        return 0;
}

static const char base64url_table[65] =
        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";

#define FSCRYPT_BASE64URL_CHARS(nbytes)        DIV_ROUND_UP((nbytes) * 4, 3)

/**
 * fscrypt_base64url_encode() - base64url-encode some binary data
 * @src: the binary data to encode
 * @srclen: the length of @src in bytes
 * @dst: (output) the base64url-encoded string.  Not NUL-terminated.
 *
 * Encodes data using base64url encoding, i.e. the "Base 64 Encoding with URL
 * and Filename Safe Alphabet" specified by RFC 4648.  '='-padding isn't used,
 * as it's unneeded and not required by the RFC.  base64url is used instead of
 * base64 to avoid the '/' character, which isn't allowed in filenames.
 *
 * Return: the length of the resulting base64url-encoded string in bytes.
 *           This will be equal to FSCRYPT_BASE64URL_CHARS(srclen).
 */
static int fscrypt_base64url_encode(const u8 *src, int srclen, char *dst)
{
        u32 ac = 0;
        int bits = 0;
        int i;
        char *cp = dst;

        for (i = 0; i < srclen; i++) {
                ac = (ac << 8) | src[i];
                bits += 8;
                do {
                        bits -= 6;
                        *cp++ = base64url_table[(ac >> bits) & 0x3f];
                } while (bits >= 6);
        }
        if (bits)
                *cp++ = base64url_table[(ac << (6 - bits)) & 0x3f];
        return cp - dst;
}

/**
 * fscrypt_base64url_decode() - base64url-decode a string
 * @src: the string to decode.  Doesn't need to be NUL-terminated.
 * @srclen: the length of @src in bytes
 * @dst: (output) the decoded binary data
 *
 * Decodes a string using base64url encoding, i.e. the "Base 64 Encoding with
 * URL and Filename Safe Alphabet" specified by RFC 4648.  '='-padding isn't
 * accepted, nor are non-encoding characters such as whitespace.
 *
 * This implementation hasn't been optimized for performance.
 *
 * Return: the length of the resulting decoded binary data in bytes,
 *           or -1 if the string isn't a valid base64url string.
 */
static int fscrypt_base64url_decode(const char *src, int srclen, u8 *dst)
{
        u32 ac = 0;
        int bits = 0;
        int i;
        u8 *bp = dst;

        for (i = 0; i < srclen; i++) {
                const char *p = strchr(base64url_table, src[i]);

                if (p == NULL || src[i] == 0)
                        return -1;
                ac = (ac << 6) | (p - base64url_table);
                bits += 6;
                if (bits >= 8) {
                        bits -= 8;
                        *bp++ = (u8)(ac >> bits);
                }
        }
        if (ac & ((1 << bits) - 1))
                return -1;
        return bp - dst;
}

bool __fscrypt_fname_encrypted_size(const union fscrypt_policy *policy,
                                    u32 orig_len, u32 max_len,
                                    u32 *encrypted_len_ret)
{
        int padding = 4 << (fscrypt_policy_flags(policy) &
                            FSCRYPT_POLICY_FLAGS_PAD_MASK);
        u32 encrypted_len;

        if (orig_len > max_len)
                return false;
        encrypted_len = max_t(u32, orig_len, FSCRYPT_FNAME_MIN_MSG_LEN);
        encrypted_len = round_up(encrypted_len, padding);
        *encrypted_len_ret = min(encrypted_len, max_len);
        return true;
}

/**
 * fscrypt_fname_encrypted_size() - calculate length of encrypted filename
 * @inode:                parent inode of dentry name being encrypted. Key must
 *                        already be set up.
 * @orig_len:                length of the original filename
 * @max_len:                maximum length to return
 * @encrypted_len_ret:        where calculated length should be returned (on success)
 *
 * Filenames that are shorter than the maximum length may have their lengths
 * increased slightly by encryption, due to padding that is applied.
 *
 * Return: false if the orig_len is greater than max_len. Otherwise, true and
 *           fill out encrypted_len_ret with the length (up to max_len).
 */
bool fscrypt_fname_encrypted_size(const struct inode *inode, u32 orig_len,
                                  u32 max_len, u32 *encrypted_len_ret)
{
        return __fscrypt_fname_encrypted_size(&inode->i_crypt_info->ci_policy,
                                              orig_len, max_len,
                                              encrypted_len_ret);
}
EXPORT_SYMBOL_GPL(fscrypt_fname_encrypted_size);

/**
 * fscrypt_fname_alloc_buffer() - allocate a buffer for presented filenames
 * @max_encrypted_len: maximum length of encrypted filenames the buffer will be
 *                       used to present
 * @crypto_str: (output) buffer to allocate
 *
 * Allocate a buffer that is large enough to hold any decrypted or encoded
 * filename (null-terminated), for the given maximum encrypted filename length.
 *
 * Return: 0 on success, -errno on failure
 */
int fscrypt_fname_alloc_buffer(u32 max_encrypted_len,
                               struct fscrypt_str *crypto_str)
{
        u32 max_presented_len = max_t(u32, FSCRYPT_NOKEY_NAME_MAX_ENCODED,
                                      max_encrypted_len);

        crypto_str->name = kmalloc(max_presented_len + 1, GFP_NOFS);
        if (!crypto_str->name)
                return -ENOMEM;
        crypto_str->len = max_presented_len;
        return 0;
}
EXPORT_SYMBOL(fscrypt_fname_alloc_buffer);

/**
 * fscrypt_fname_free_buffer() - free a buffer for presented filenames
 * @crypto_str: the buffer to free
 *
 * Free a buffer that was allocated by fscrypt_fname_alloc_buffer().
 */
void fscrypt_fname_free_buffer(struct fscrypt_str *crypto_str)
{
        if (!crypto_str)
                return;
        kfree(crypto_str->name);
        crypto_str->name = NULL;
}
EXPORT_SYMBOL(fscrypt_fname_free_buffer);

/**
 * fscrypt_fname_disk_to_usr() - convert an encrypted filename to
 *                                 user-presentable form
 * @inode: inode of the parent directory (for regular filenames)
 *           or of the symlink (for symlink targets)
 * @hash: first part of the name's dirhash, if applicable.  This only needs to
 *          be provided if the filename is located in an indexed directory whose
 *          encryption key may be unavailable.  Not needed for symlink targets.
 * @minor_hash: second part of the name's dirhash, if applicable
 * @iname: encrypted filename to convert.  May also be "." or "..", which
 *           aren't actually encrypted.
 * @oname: output buffer for the user-presentable filename.  The caller must
 *           have allocated enough space for this, e.g. using
 *           fscrypt_fname_alloc_buffer().
 *
 * If the key is available, we'll decrypt the disk name.  Otherwise, we'll
 * encode it for presentation in fscrypt_nokey_name format.
 * See struct fscrypt_nokey_name for details.
 *
 * Return: 0 on success, -errno on failure
 */
int fscrypt_fname_disk_to_usr(const struct inode *inode,
                              u32 hash, u32 minor_hash,
                              const struct fscrypt_str *iname,
                              struct fscrypt_str *oname)
{
        const struct qstr qname = FSTR_TO_QSTR(iname);
        struct fscrypt_nokey_name nokey_name;
        u32 size; /* size of the unencoded no-key name */

        if (fscrypt_is_dot_dotdot(&qname)) {
                oname->name[0] = '.';
                oname->name[iname->len - 1] = '.';
                oname->len = iname->len;
                return 0;
        }

        if (iname->len < FSCRYPT_FNAME_MIN_MSG_LEN)
                return -EUCLEAN;

        if (fscrypt_has_encryption_key(inode))
                return fname_decrypt(inode, iname, oname);

        /*
         * Sanity check that struct fscrypt_nokey_name doesn't have padding
         * between fields and that its encoded size never exceeds NAME_MAX.
         */
        BUILD_BUG_ON(offsetofend(struct fscrypt_nokey_name, dirhash) !=
                     offsetof(struct fscrypt_nokey_name, bytes));
        BUILD_BUG_ON(offsetofend(struct fscrypt_nokey_name, bytes) !=
                     offsetof(struct fscrypt_nokey_name, sha256));
        BUILD_BUG_ON(FSCRYPT_NOKEY_NAME_MAX_ENCODED > NAME_MAX);

        nokey_name.dirhash[0] = hash;
        nokey_name.dirhash[1] = minor_hash;

        if (iname->len <= sizeof(nokey_name.bytes)) {
                memcpy(nokey_name.bytes, iname->name, iname->len);
                size = offsetof(struct fscrypt_nokey_name, bytes[iname->len]);
        } else {
                memcpy(nokey_name.bytes, iname->name, sizeof(nokey_name.bytes));
                /* Compute strong hash of remaining part of name. */
                sha256(&iname->name[sizeof(nokey_name.bytes)],
                       iname->len - sizeof(nokey_name.bytes),
                       nokey_name.sha256);
                size = FSCRYPT_NOKEY_NAME_MAX;
        }
        oname->len = fscrypt_base64url_encode((const u8 *)&nokey_name, size,
                                              oname->name);
        return 0;
}
EXPORT_SYMBOL(fscrypt_fname_disk_to_usr);

/**
 * fscrypt_setup_filename() - prepare to search a possibly encrypted directory
 * @dir: the directory that will be searched
 * @iname: the user-provided filename being searched for
 * @lookup: 1 if we're allowed to proceed without the key because it's
 *        ->lookup() or we're finding the dir_entry for deletion; 0 if we cannot
 *        proceed without the key because we're going to create the dir_entry.
 * @fname: the filename information to be filled in
 *
 * Given a user-provided filename @iname, this function sets @fname->disk_name
 * to the name that would be stored in the on-disk directory entry, if possible.
 * If the directory is unencrypted this is simply @iname.  Else, if we have the
 * directory's encryption key, then @iname is the plaintext, so we encrypt it to
 * get the disk_name.
 *
 * Else, for keyless @lookup operations, @iname should be a no-key name, so we
 * decode it to get the struct fscrypt_nokey_name.  Non-@lookup operations will
 * be impossible in this case, so we fail them with ENOKEY.
 *
 * If successful, fscrypt_free_filename() must be called later to clean up.
 *
 * Return: 0 on success, -errno on failure
 */
int fscrypt_setup_filename(struct inode *dir, const struct qstr *iname,
                              int lookup, struct fscrypt_name *fname)
{
        struct fscrypt_nokey_name *nokey_name;
        int ret;

        memset(fname, 0, sizeof(struct fscrypt_name));
        fname->usr_fname = iname;

        if (!IS_ENCRYPTED(dir) || fscrypt_is_dot_dotdot(iname)) {
                fname->disk_name.name = (unsigned char *)iname->name;
                fname->disk_name.len = iname->len;
                return 0;
        }
        ret = fscrypt_get_encryption_info(dir, lookup);
        if (ret)
                return ret;

        if (fscrypt_has_encryption_key(dir)) {
                if (!fscrypt_fname_encrypted_size(dir, iname->len, NAME_MAX,
                                                  &fname->crypto_buf.len))
                        return -ENAMETOOLONG;
                fname->crypto_buf.name = kmalloc(fname->crypto_buf.len,
                                                 GFP_NOFS);
                if (!fname->crypto_buf.name)
                        return -ENOMEM;

                ret = fscrypt_fname_encrypt(dir, iname, fname->crypto_buf.name,
                                            fname->crypto_buf.len);
                if (ret)
                        goto errout;
                fname->disk_name.name = fname->crypto_buf.name;
                fname->disk_name.len = fname->crypto_buf.len;
                return 0;
        }
        if (!lookup)
                return -ENOKEY;
        fname->is_nokey_name = true;

        /*
         * We don't have the key and we are doing a lookup; decode the
         * user-supplied name
         */

        if (iname->len > FSCRYPT_NOKEY_NAME_MAX_ENCODED)
                return -ENOENT;

        fname->crypto_buf.name = kmalloc(FSCRYPT_NOKEY_NAME_MAX, GFP_KERNEL);
        if (fname->crypto_buf.name == NULL)
                return -ENOMEM;

        ret = fscrypt_base64url_decode(iname->name, iname->len,
                                       fname->crypto_buf.name);
        if (ret < (int)offsetof(struct fscrypt_nokey_name, bytes[1]) ||
            (ret > offsetof(struct fscrypt_nokey_name, sha256) &&
             ret != FSCRYPT_NOKEY_NAME_MAX)) {
                ret = -ENOENT;
                goto errout;
        }
        fname->crypto_buf.len = ret;

        nokey_name = (void *)fname->crypto_buf.name;
        fname->hash = nokey_name->dirhash[0];
        fname->minor_hash = nokey_name->dirhash[1];
        if (ret != FSCRYPT_NOKEY_NAME_MAX) {
                /* The full ciphertext filename is available. */
                fname->disk_name.name = nokey_name->bytes;
                fname->disk_name.len =
                        ret - offsetof(struct fscrypt_nokey_name, bytes);
        }
        return 0;

errout:
        kfree(fname->crypto_buf.name);
        return ret;
}
EXPORT_SYMBOL(fscrypt_setup_filename);

/**
 * fscrypt_match_name() - test whether the given name matches a directory entry
 * @fname: the name being searched for
 * @de_name: the name from the directory entry
 * @de_name_len: the length of @de_name in bytes
 *
 * Normally @fname->disk_name will be set, and in that case we simply compare
 * that to the name stored in the directory entry.  The only exception is that
 * if we don't have the key for an encrypted directory and the name we're
 * looking for is very long, then we won't have the full disk_name and instead
 * we'll need to match against a fscrypt_nokey_name that includes a strong hash.
 *
 * Return: %true if the name matches, otherwise %false.
 */
bool fscrypt_match_name(const struct fscrypt_name *fname,
                        const u8 *de_name, u32 de_name_len)
{
        const struct fscrypt_nokey_name *nokey_name =
                (const void *)fname->crypto_buf.name;
        u8 digest[SHA256_DIGEST_SIZE];

        if (likely(fname->disk_name.name)) {
                if (de_name_len != fname->disk_name.len)
                        return false;
                return !memcmp(de_name, fname->disk_name.name, de_name_len);
        }
        if (de_name_len <= sizeof(nokey_name->bytes))
                return false;
        if (memcmp(de_name, nokey_name->bytes, sizeof(nokey_name->bytes)))
                return false;
        sha256(&de_name[sizeof(nokey_name->bytes)],
               de_name_len - sizeof(nokey_name->bytes), digest);
        return !memcmp(digest, nokey_name->sha256, sizeof(digest));
}
EXPORT_SYMBOL_GPL(fscrypt_match_name);

/**
 * fscrypt_fname_siphash() - calculate the SipHash of a filename
 * @dir: the parent directory
 * @name: the filename to calculate the SipHash of
 *
 * Given a plaintext filename @name and a directory @dir which uses SipHash as
 * its dirhash method and has had its fscrypt key set up, this function
 * calculates the SipHash of that name using the directory's secret dirhash key.
 *
 * Return: the SipHash of @name using the hash key of @dir
 */
u64 fscrypt_fname_siphash(const struct inode *dir, const struct qstr *name)
{
        const struct fscrypt_inode_info *ci = dir->i_crypt_info;

        WARN_ON_ONCE(!ci->ci_dirhash_key_initialized);

        return siphash(name->name, name->len, &ci->ci_dirhash_key);
}
EXPORT_SYMBOL_GPL(fscrypt_fname_siphash);

/*
 * Validate dentries in encrypted directories to make sure we aren't potentially
 * caching stale dentries after a key has been added.
 */
int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags)
{
        struct dentry *dir;
        int err;
        int valid;

        /*
         * Plaintext names are always valid, since fscrypt doesn't support
         * reverting to no-key names without evicting the directory's inode
         * -- which implies eviction of the dentries in the directory.
         */
        if (!(dentry->d_flags & DCACHE_NOKEY_NAME))
                return 1;

        /*
         * No-key name; valid if the directory's key is still unavailable.
         *
         * Although fscrypt forbids rename() on no-key names, we still must use
         * dget_parent() here rather than use ->d_parent directly.  That's
         * because a corrupted fs image may contain directory hard links, which
         * the VFS handles by moving the directory's dentry tree in the dcache
         * each time ->lookup() finds the directory and it already has a dentry
         * elsewhere.  Thus ->d_parent can be changing, and we must safely grab
         * a reference to some ->d_parent to prevent it from being freed.
         */

        if (flags & LOOKUP_RCU)
                return -ECHILD;

        dir = dget_parent(dentry);
        /*
         * Pass allow_unsupported=true, so that files with an unsupported
         * encryption policy can be deleted.
         */
        err = fscrypt_get_encryption_info(d_inode(dir), true);
        valid = !fscrypt_has_encryption_key(d_inode(dir));
        dput(dir);

        if (err < 0)
                return err;

        return valid;
}
EXPORT_SYMBOL_GPL(fscrypt_d_revalidate);


































































  156 
  155 
  155 




  156 





























































































































































  156 












  156 






































































  155 









































  156 

  156 






































  156 




   75 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Link physical devices with ACPI devices support
 *
 * Copyright (c) 2005 David Shaohua Li <shaohua.li@intel.com>
 * Copyright (c) 2005 Intel Corp.
 */

#define pr_fmt(fmt) "ACPI: " fmt

#include <linux/acpi_iort.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/rwsem.h>
#include <linux/acpi.h>
#include <linux/dma-mapping.h>
#include <linux/pci.h>
#include <linux/pci-acpi.h>
#include <linux/platform_device.h>

#include "internal.h"

static LIST_HEAD(bus_type_list);
static DECLARE_RWSEM(bus_type_sem);

#define PHYSICAL_NODE_STRING "physical_node"
#define PHYSICAL_NODE_NAME_SIZE (sizeof(PHYSICAL_NODE_STRING) + 10)

int register_acpi_bus_type(struct acpi_bus_type *type)
{
        if (acpi_disabled)
                return -ENODEV;
        if (type && type->match && type->find_companion) {
                down_write(&bus_type_sem);
                list_add_tail(&type->list, &bus_type_list);
                up_write(&bus_type_sem);
                pr_info("bus type %s registered\n", type->name);
                return 0;
        }
        return -ENODEV;
}
EXPORT_SYMBOL_GPL(register_acpi_bus_type);

int unregister_acpi_bus_type(struct acpi_bus_type *type)
{
        if (acpi_disabled)
                return 0;
        if (type) {
                down_write(&bus_type_sem);
                list_del_init(&type->list);
                up_write(&bus_type_sem);
                pr_info("bus type %s unregistered\n", type->name);
                return 0;
        }
        return -ENODEV;
}
EXPORT_SYMBOL_GPL(unregister_acpi_bus_type);

static struct acpi_bus_type *acpi_get_bus_type(struct device *dev)
{
        struct acpi_bus_type *tmp, *ret = NULL;

        down_read(&bus_type_sem);
        list_for_each_entry(tmp, &bus_type_list, list) {
                if (tmp->match(dev)) {
                        ret = tmp;
                        break;
                }
        }
        up_read(&bus_type_sem);
        return ret;
}

#define FIND_CHILD_MIN_SCORE        1
#define FIND_CHILD_MID_SCORE        2
#define FIND_CHILD_MAX_SCORE        3

static int match_any(struct acpi_device *adev, void *not_used)
{
        return 1;
}

static bool acpi_dev_has_children(struct acpi_device *adev)
{
        return acpi_dev_for_each_child(adev, match_any, NULL) > 0;
}

static int find_child_checks(struct acpi_device *adev, bool check_children)
{
        unsigned long long sta;
        acpi_status status;

        if (check_children && !acpi_dev_has_children(adev))
                return -ENODEV;

        status = acpi_evaluate_integer(adev->handle, "_STA", NULL, &sta);
        if (status == AE_NOT_FOUND) {
                /*
                 * Special case: backlight device objects without _STA are
                 * preferred to other objects with the same _ADR value, because
                 * it is more likely that they are actually useful.
                 */
                if (adev->pnp.type.backlight)
                        return FIND_CHILD_MID_SCORE;

                return FIND_CHILD_MIN_SCORE;
        }

        if (ACPI_FAILURE(status) || !(sta & ACPI_STA_DEVICE_ENABLED))
                return -ENODEV;

        /*
         * If the device has a _HID returning a valid ACPI/PNP device ID, it is
         * better to make it look less attractive here, so that the other device
         * with the same _ADR value (that may not have a valid device ID) can be
         * matched going forward.  [This means a second spec violation in a row,
         * so whatever we do here is best effort anyway.]
         */
        if (adev->pnp.type.platform_id)
                return FIND_CHILD_MIN_SCORE;

        return FIND_CHILD_MAX_SCORE;
}

struct find_child_walk_data {
        struct acpi_device *adev;
        u64 address;
        int score;
        bool check_sta;
        bool check_children;
};

static int check_one_child(struct acpi_device *adev, void *data)
{
        struct find_child_walk_data *wd = data;
        int score;

        if (!adev->pnp.type.bus_address || acpi_device_adr(adev) != wd->address)
                return 0;

        if (!wd->adev) {
                /*
                 * This is the first matching object, so save it.  If it is not
                 * necessary to look for any other matching objects, stop the
                 * search.
                 */
                wd->adev = adev;
                return !(wd->check_sta || wd->check_children);
        }

        /*
         * There is more than one matching device object with the same _ADR
         * value.  That really is unexpected, so we are kind of beyond the scope
         * of the spec here.  We have to choose which one to return, though.
         *
         * First, get the score for the previously found object and terminate
         * the walk if it is maximum.
        */
        if (!wd->score) {
                score = find_child_checks(wd->adev, wd->check_children);
                if (score == FIND_CHILD_MAX_SCORE)
                        return 1;

                wd->score = score;
        }
        /*
         * Second, if the object that has just been found has a better score,
         * replace the previously found one with it and terminate the walk if
         * the new score is maximum.
         */
        score = find_child_checks(adev, wd->check_children);
        if (score > wd->score) {
                wd->adev = adev;
                if (score == FIND_CHILD_MAX_SCORE)
                        return 1;

                wd->score = score;
        }

        /* Continue, because there may be better matches. */
        return 0;
}

static struct acpi_device *acpi_find_child(struct acpi_device *parent,
                                           u64 address, bool check_children,
                                           bool check_sta)
{
        struct find_child_walk_data wd = {
                .address = address,
                .check_children = check_children,
                .check_sta = check_sta,
                .adev = NULL,
                .score = 0,
        };

        if (parent)
                acpi_dev_for_each_child(parent, check_one_child, &wd);

        return wd.adev;
}

struct acpi_device *acpi_find_child_device(struct acpi_device *parent,
                                           u64 address, bool check_children)
{
        return acpi_find_child(parent, address, check_children, true);
}
EXPORT_SYMBOL_GPL(acpi_find_child_device);

struct acpi_device *acpi_find_child_by_adr(struct acpi_device *adev,
                                           acpi_bus_address adr)
{
        return acpi_find_child(adev, adr, false, false);
}
EXPORT_SYMBOL_GPL(acpi_find_child_by_adr);

static void acpi_physnode_link_name(char *buf, unsigned int node_id)
{
        if (node_id > 0)
                snprintf(buf, PHYSICAL_NODE_NAME_SIZE,
                         PHYSICAL_NODE_STRING "%u", node_id);
        else
                strcpy(buf, PHYSICAL_NODE_STRING);
}

int acpi_bind_one(struct device *dev, struct acpi_device *acpi_dev)
{
        struct acpi_device_physical_node *physical_node, *pn;
        char physical_node_name[PHYSICAL_NODE_NAME_SIZE];
        struct list_head *physnode_list;
        unsigned int node_id;
        int retval = -EINVAL;

        if (has_acpi_companion(dev)) {
                if (acpi_dev) {
                        dev_warn(dev, "ACPI companion already set\n");
                        return -EINVAL;
                } else {
                        acpi_dev = ACPI_COMPANION(dev);
                }
        }
        if (!acpi_dev)
                return -EINVAL;

        acpi_dev_get(acpi_dev);
        get_device(dev);
        physical_node = kzalloc(sizeof(*physical_node), GFP_KERNEL);
        if (!physical_node) {
                retval = -ENOMEM;
                goto err;
        }

        mutex_lock(&acpi_dev->physical_node_lock);

        /*
         * Keep the list sorted by node_id so that the IDs of removed nodes can
         * be recycled easily.
         */
        physnode_list = &acpi_dev->physical_node_list;
        node_id = 0;
        list_for_each_entry(pn, &acpi_dev->physical_node_list, node) {
                /* Sanity check. */
                if (pn->dev == dev) {
                        mutex_unlock(&acpi_dev->physical_node_lock);

                        dev_warn(dev, "Already associated with ACPI node\n");
                        kfree(physical_node);
                        if (ACPI_COMPANION(dev) != acpi_dev)
                                goto err;

                        put_device(dev);
                        acpi_dev_put(acpi_dev);
                        return 0;
                }
                if (pn->node_id == node_id) {
                        physnode_list = &pn->node;
                        node_id++;
                }
        }

        physical_node->node_id = node_id;
        physical_node->dev = dev;
        list_add(&physical_node->node, physnode_list);
        acpi_dev->physical_node_count++;

        if (!has_acpi_companion(dev))
                ACPI_COMPANION_SET(dev, acpi_dev);

        acpi_physnode_link_name(physical_node_name, node_id);
        retval = sysfs_create_link(&acpi_dev->dev.kobj, &dev->kobj,
                                   physical_node_name);
        if (retval)
                dev_err(&acpi_dev->dev, "Failed to create link %s (%d)\n",
                        physical_node_name, retval);

        retval = sysfs_create_link(&dev->kobj, &acpi_dev->dev.kobj,
                                   "firmware_node");
        if (retval)
                dev_err(dev, "Failed to create link firmware_node (%d)\n",
                        retval);

        mutex_unlock(&acpi_dev->physical_node_lock);

        if (acpi_dev->wakeup.flags.valid)
                device_set_wakeup_capable(dev, true);

        return 0;

 err:
        ACPI_COMPANION_SET(dev, NULL);
        put_device(dev);
        acpi_dev_put(acpi_dev);
        return retval;
}
EXPORT_SYMBOL_GPL(acpi_bind_one);

int acpi_unbind_one(struct device *dev)
{
        struct acpi_device *acpi_dev = ACPI_COMPANION(dev);
        struct acpi_device_physical_node *entry;

        if (!acpi_dev)
                return 0;

        mutex_lock(&acpi_dev->physical_node_lock);

        list_for_each_entry(entry, &acpi_dev->physical_node_list, node)
                if (entry->dev == dev) {
                        char physnode_name[PHYSICAL_NODE_NAME_SIZE];

                        list_del(&entry->node);
                        acpi_dev->physical_node_count--;

                        acpi_physnode_link_name(physnode_name, entry->node_id);
                        sysfs_remove_link(&acpi_dev->dev.kobj, physnode_name);
                        sysfs_remove_link(&dev->kobj, "firmware_node");
                        ACPI_COMPANION_SET(dev, NULL);
                        /* Drop references taken by acpi_bind_one(). */
                        put_device(dev);
                        acpi_dev_put(acpi_dev);
                        kfree(entry);
                        break;
                }

        mutex_unlock(&acpi_dev->physical_node_lock);
        return 0;
}
EXPORT_SYMBOL_GPL(acpi_unbind_one);

void acpi_device_notify(struct device *dev)
{
        struct acpi_device *adev;
        int ret;

        ret = acpi_bind_one(dev, NULL);
        if (ret) {
                struct acpi_bus_type *type = acpi_get_bus_type(dev);

                if (!type)
                        goto err;

                adev = type->find_companion(dev);
                if (!adev) {
                        dev_dbg(dev, "ACPI companion not found\n");
                        goto err;
                }
                ret = acpi_bind_one(dev, adev);
                if (ret)
                        goto err;

                if (type->setup) {
                        type->setup(dev);
                        goto done;
                }
        } else {
                adev = ACPI_COMPANION(dev);

                if (dev_is_pci(dev)) {
                        pci_acpi_setup(dev, adev);
                        goto done;
                } else if (dev_is_platform(dev)) {
                        acpi_configure_pmsi_domain(dev);
                }
        }

        if (adev->handler && adev->handler->bind)
                adev->handler->bind(dev);

done:
        acpi_handle_debug(ACPI_HANDLE(dev), "Bound to device %s\n",
                          dev_name(dev));

        return;

err:
        dev_dbg(dev, "No ACPI support\n");
}

void acpi_device_notify_remove(struct device *dev)
{
        struct acpi_device *adev = ACPI_COMPANION(dev);

        if (!adev)
                return;

        if (dev_is_pci(dev))
                pci_acpi_cleanup(dev, adev);
        else if (adev->handler && adev->handler->unbind)
                adev->handler->unbind(dev);

        acpi_unbind_one(dev);
}





















   94 































   68 


   68 

   68 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#include <linux/notifier.h>
#include <linux/socket.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <net/net_namespace.h>
#include <net/fib_notifier.h>
#include <net/netns/ipv6.h>
#include <net/ip6_fib.h>

int call_fib6_notifier(struct notifier_block *nb,
                       enum fib_event_type event_type,
                       struct fib_notifier_info *info)
{
        info->family = AF_INET6;
        return call_fib_notifier(nb, event_type, info);
}

int call_fib6_notifiers(struct net *net, enum fib_event_type event_type,
                        struct fib_notifier_info *info)
{
        info->family = AF_INET6;
        return call_fib_notifiers(net, event_type, info);
}

static unsigned int fib6_seq_read(const struct net *net)
{
        return fib6_tables_seq_read(net) + fib6_rules_seq_read(net);
}

static int fib6_dump(struct net *net, struct notifier_block *nb,
                     struct netlink_ext_ack *extack)
{
        int err;

        err = fib6_rules_dump(net, nb, extack);
        if (err)
                return err;

        return fib6_tables_dump(net, nb, extack);
}

static const struct fib_notifier_ops fib6_notifier_ops_template = {
        .family                = AF_INET6,
        .fib_seq_read        = fib6_seq_read,
        .fib_dump        = fib6_dump,
        .owner                = THIS_MODULE,
};

int __net_init fib6_notifier_init(struct net *net)
{
        struct fib_notifier_ops *ops;

        ops = fib_notifier_ops_register(&fib6_notifier_ops_template, net);
        if (IS_ERR(ops))
                return PTR_ERR(ops);
        net->ipv6.notifier_ops = ops;

        return 0;
}

void __net_exit fib6_notifier_exit(struct net *net)
{
        fib_notifier_ops_unregister(net->ipv6.notifier_ops);
}





























































































































































































  280 













1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef MPLS_INTERNAL_H
#define MPLS_INTERNAL_H
#include <net/mpls.h>

/* put a reasonable limit on the number of labels
 * we will accept from userspace
 */
#define MAX_NEW_LABELS 30

struct mpls_entry_decoded {
        u32 label;
        u8 ttl;
        u8 tc;
        u8 bos;
};

struct mpls_pcpu_stats {
        struct mpls_link_stats        stats;
        struct u64_stats_sync        syncp;
};

struct mpls_dev {
        int                                input_enabled;
        struct net_device                *dev;
        struct mpls_pcpu_stats __percpu        *stats;

        struct ctl_table_header                *sysctl;
        struct rcu_head                        rcu;
};

#if BITS_PER_LONG == 32

#define MPLS_INC_STATS_LEN(mdev, len, pkts_field, bytes_field)                \
        do {                                                                \
                __typeof__(*(mdev)->stats) *ptr =                        \
                        raw_cpu_ptr((mdev)->stats);                        \
                local_bh_disable();                                        \
                u64_stats_update_begin(&ptr->syncp);                        \
                ptr->stats.pkts_field++;                                \
                ptr->stats.bytes_field += (len);                        \
                u64_stats_update_end(&ptr->syncp);                        \
                local_bh_enable();                                        \
        } while (0)

#define MPLS_INC_STATS(mdev, field)                                        \
        do {                                                                \
                __typeof__(*(mdev)->stats) *ptr =                        \
                        raw_cpu_ptr((mdev)->stats);                        \
                local_bh_disable();                                        \
                u64_stats_update_begin(&ptr->syncp);                        \
                ptr->stats.field++;                                        \
                u64_stats_update_end(&ptr->syncp);                        \
                local_bh_enable();                                        \
        } while (0)

#else

#define MPLS_INC_STATS_LEN(mdev, len, pkts_field, bytes_field)                \
        do {                                                                \
                this_cpu_inc((mdev)->stats->stats.pkts_field);                \
                this_cpu_add((mdev)->stats->stats.bytes_field, (len));        \
        } while (0)

#define MPLS_INC_STATS(mdev, field)                        \
        this_cpu_inc((mdev)->stats->stats.field)

#endif

struct sk_buff;

#define LABEL_NOT_SPECIFIED (1 << 20)

/* This maximum ha length copied from the definition of struct neighbour */
#define VIA_ALEN_ALIGN sizeof(unsigned long)
#define MAX_VIA_ALEN (ALIGN(MAX_ADDR_LEN, VIA_ALEN_ALIGN))

enum mpls_payload_type {
        MPT_UNSPEC, /* IPv4 or IPv6 */
        MPT_IPV4 = 4,
        MPT_IPV6 = 6,

        /* Other types not implemented:
         *  - Pseudo-wire with or without control word (RFC4385)
         *  - GAL (RFC5586)
         */
};

struct mpls_nh { /* next hop label forwarding entry */
        struct net_device        *nh_dev;

        /* nh_flags is accessed under RCU in the packet path; it is
         * modified handling netdev events with rtnl lock held
         */
        unsigned int                nh_flags;
        u8                        nh_labels;
        u8                        nh_via_alen;
        u8                        nh_via_table;
        u8                        nh_reserved1;

        u32                        nh_label[];
};

/* offset of via from beginning of mpls_nh */
#define MPLS_NH_VIA_OFF(num_labels) \
                ALIGN(sizeof(struct mpls_nh) + (num_labels) * sizeof(u32), \
                      VIA_ALEN_ALIGN)

/* all nexthops within a route have the same size based on the
 * max number of labels and max via length across all nexthops
 */
#define MPLS_NH_SIZE(num_labels, max_via_alen)                \
                (MPLS_NH_VIA_OFF((num_labels)) +        \
                ALIGN((max_via_alen), VIA_ALEN_ALIGN))

enum mpls_ttl_propagation {
        MPLS_TTL_PROP_DEFAULT,
        MPLS_TTL_PROP_ENABLED,
        MPLS_TTL_PROP_DISABLED,
};

/* The route, nexthops and vias are stored together in the same memory
 * block:
 *
 * +----------------------+
 * | mpls_route           |
 * +----------------------+
 * | mpls_nh 0            |
 * +----------------------+
 * | alignment padding    |   4 bytes for odd number of labels
 * +----------------------+
 * | via[rt_max_alen] 0   |
 * +----------------------+
 * | alignment padding    |   via's aligned on sizeof(unsigned long)
 * +----------------------+
 * | ...                  |
 * +----------------------+
 * | mpls_nh n-1          |
 * +----------------------+
 * | via[rt_max_alen] n-1 |
 * +----------------------+
 */
struct mpls_route { /* next hop label forwarding entry */
        struct rcu_head                rt_rcu;
        u8                        rt_protocol;
        u8                        rt_payload_type;
        u8                        rt_max_alen;
        u8                        rt_ttl_propagate;
        u8                        rt_nhn;
        /* rt_nhn_alive is accessed under RCU in the packet path; it
         * is modified handling netdev events with rtnl lock held
         */
        u8                        rt_nhn_alive;
        u8                        rt_nh_size;
        u8                        rt_via_offset;
        u8                        rt_reserved1;
        struct mpls_nh                rt_nh[];
};

#define for_nexthops(rt) {                                                \
        int nhsel; const struct mpls_nh *nh;                                \
        for (nhsel = 0, nh = (rt)->rt_nh;                                \
             nhsel < (rt)->rt_nhn;                                        \
             nh = (void *)nh + (rt)->rt_nh_size, nhsel++)

#define change_nexthops(rt) {                                                \
        int nhsel; struct mpls_nh *nh;                                        \
        for (nhsel = 0, nh = (rt)->rt_nh;                                \
             nhsel < (rt)->rt_nhn;                                        \
             nh = (void *)nh + (rt)->rt_nh_size, nhsel++)

#define endfor_nexthops(rt) }

static inline struct mpls_entry_decoded mpls_entry_decode(struct mpls_shim_hdr *hdr)
{
        struct mpls_entry_decoded result;
        unsigned entry = be32_to_cpu(hdr->label_stack_entry);

        result.label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT;
        result.ttl = (entry & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
        result.tc =  (entry & MPLS_LS_TC_MASK) >> MPLS_LS_TC_SHIFT;
        result.bos = (entry & MPLS_LS_S_MASK) >> MPLS_LS_S_SHIFT;

        return result;
}

static inline struct mpls_dev *mpls_dev_get(const struct net_device *dev)
{
        return rcu_dereference_rtnl(dev->mpls_ptr);
}

int nla_put_labels(struct sk_buff *skb, int attrtype,  u8 labels,
                   const u32 label[]);
int nla_get_labels(const struct nlattr *nla, u8 max_labels, u8 *labels,
                   u32 label[], struct netlink_ext_ack *extack);
bool mpls_output_possible(const struct net_device *dev);
unsigned int mpls_dev_mtu(const struct net_device *dev);
bool mpls_pkt_too_big(const struct sk_buff *skb, unsigned int mtu);
void mpls_stats_inc_outucastpkts(struct net_device *dev,
                                 const struct sk_buff *skb);

#endif /* MPLS_INTERNAL_H */







































































  116 








































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_SMP_H
#define __LINUX_SMP_H

/*
 *        Generic SMP support
 *                Alan Cox. <alan@redhat.com>
 */

#include <linux/errno.h>
#include <linux/types.h>
#include <linux/list.h>
#include <linux/cpumask.h>
#include <linux/init.h>
#include <linux/smp_types.h>

typedef void (*smp_call_func_t)(void *info);
typedef bool (*smp_cond_func_t)(int cpu, void *info);

/*
 * structure shares (partial) layout with struct irq_work
 */
struct __call_single_data {
        struct __call_single_node node;
        smp_call_func_t func;
        void *info;
};

#define CSD_INIT(_func, _info) \
        (struct __call_single_data){ .func = (_func), .info = (_info), }

/* Use __aligned() to avoid to use 2 cache lines for 1 csd */
typedef struct __call_single_data call_single_data_t
        __aligned(sizeof(struct __call_single_data));

#define INIT_CSD(_csd, _func, _info)                \
do {                                                \
        *(_csd) = CSD_INIT((_func), (_info));        \
} while (0)

/*
 * Enqueue a llist_node on the call_single_queue; be very careful, read
 * flush_smp_call_function_queue() in detail.
 */
extern void __smp_call_single_queue(int cpu, struct llist_node *node);

/* total number of cpus in this system (may exceed NR_CPUS) */
extern unsigned int total_cpus;

int smp_call_function_single(int cpuid, smp_call_func_t func, void *info,
                             int wait);

void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
                           void *info, bool wait, const struct cpumask *mask);

int smp_call_function_single_async(int cpu, call_single_data_t *csd);

/*
 * Cpus stopping functions in panic. All have default weak definitions.
 * Architecture-dependent code may override them.
 */
void __noreturn panic_smp_self_stop(void);
void __noreturn nmi_panic_self_stop(struct pt_regs *regs);
void crash_smp_send_stop(void);

/*
 * Call a function on all processors
 */
static inline void on_each_cpu(smp_call_func_t func, void *info, int wait)
{
        on_each_cpu_cond_mask(NULL, func, info, wait, cpu_online_mask);
}

/**
 * on_each_cpu_mask(): Run a function on processors specified by
 * cpumask, which may include the local processor.
 * @mask: The set of cpus to run on (only runs on online subset).
 * @func: The function to run. This must be fast and non-blocking.
 * @info: An arbitrary pointer to pass to the function.
 * @wait: If true, wait (atomically) until function has completed
 *        on other CPUs.
 *
 * If @wait is true, then returns once @func has returned.
 *
 * You must not call this function with disabled interrupts or from a
 * hardware interrupt handler or from a bottom half handler.  The
 * exception is that it may be used during early boot while
 * early_boot_irqs_disabled is set.
 */
static inline void on_each_cpu_mask(const struct cpumask *mask,
                                    smp_call_func_t func, void *info, bool wait)
{
        on_each_cpu_cond_mask(NULL, func, info, wait, mask);
}

/*
 * Call a function on each processor for which the supplied function
 * cond_func returns a positive value. This may include the local
 * processor.  May be used during early boot while early_boot_irqs_disabled is
 * set. Use local_irq_save/restore() instead of local_irq_disable/enable().
 */
static inline void on_each_cpu_cond(smp_cond_func_t cond_func,
                                    smp_call_func_t func, void *info, bool wait)
{
        on_each_cpu_cond_mask(cond_func, func, info, wait, cpu_online_mask);
}

/*
 * Architecture specific boot CPU setup.  Defined as empty weak function in
 * init/main.c. Architectures can override it.
 */
void __init smp_prepare_boot_cpu(void);

#ifdef CONFIG_SMP

#include <linux/preempt.h>
#include <linux/compiler.h>
#include <linux/thread_info.h>
#include <asm/smp.h>

/*
 * main cross-CPU interfaces, handles INIT, TLB flush, STOP, etc.
 * (defined in asm header):
 */

/*
 * stops all CPUs but the current one:
 */
extern void smp_send_stop(void);

/*
 * sends a 'reschedule' event to another CPU:
 */
extern void arch_smp_send_reschedule(int cpu);
/*
 * scheduler_ipi() is inline so can't be passed as callback reason, but the
 * callsite IP should be sufficient for root-causing IPIs sent from here.
 */
#define smp_send_reschedule(cpu) ({                  \
        trace_ipi_send_cpu(cpu, _RET_IP_, NULL);  \
        arch_smp_send_reschedule(cpu);                  \
})

/*
 * Prepare machine for booting other CPUs.
 */
extern void smp_prepare_cpus(unsigned int max_cpus);

/*
 * Bring a CPU up
 */
extern int __cpu_up(unsigned int cpunum, struct task_struct *tidle);

/*
 * Final polishing of CPUs
 */
extern void smp_cpus_done(unsigned int max_cpus);

/*
 * Call a function on all other processors
 */
void smp_call_function(smp_call_func_t func, void *info, int wait);
void smp_call_function_many(const struct cpumask *mask,
                            smp_call_func_t func, void *info, bool wait);

int smp_call_function_any(const struct cpumask *mask,
                          smp_call_func_t func, void *info, int wait);

void kick_all_cpus_sync(void);
void wake_up_all_idle_cpus(void);

/*
 * Generic and arch helpers
 */
void __init call_function_init(void);
void generic_smp_call_function_single_interrupt(void);
#define generic_smp_call_function_interrupt \
        generic_smp_call_function_single_interrupt

extern unsigned int setup_max_cpus;
extern void __init setup_nr_cpu_ids(void);
extern void __init smp_init(void);

extern int __boot_cpu_id;

static inline int get_boot_cpu_id(void)
{
        return __boot_cpu_id;
}

#else /* !SMP */

static inline void smp_send_stop(void) { }

/*
 *        These macros fold the SMP functionality into a single CPU system
 */
#define raw_smp_processor_id()                        0
static inline void up_smp_call_function(smp_call_func_t func, void *info)
{
}
#define smp_call_function(func, info, wait) \
                        (up_smp_call_function(func, info))

static inline void smp_send_reschedule(int cpu) { }
#define smp_call_function_many(mask, func, info, wait) \
                        (up_smp_call_function(func, info))
static inline void call_function_init(void) { }

static inline int
smp_call_function_any(const struct cpumask *mask, smp_call_func_t func,
                      void *info, int wait)
{
        return smp_call_function_single(0, func, info, wait);
}

static inline void kick_all_cpus_sync(void) {  }
static inline void wake_up_all_idle_cpus(void) {  }

#define setup_max_cpus 0

#ifdef CONFIG_UP_LATE_INIT
extern void __init up_late_init(void);
static inline void smp_init(void) { up_late_init(); }
#else
static inline void smp_init(void) { }
#endif

static inline int get_boot_cpu_id(void)
{
        return 0;
}

#endif /* !SMP */

/**
 * raw_processor_id() - get the current (unstable) CPU id
 *
 * For then you know what you are doing and need an unstable
 * CPU id.
 */

/**
 * smp_processor_id() - get the current (stable) CPU id
 *
 * This is the normal accessor to the CPU id and should be used
 * whenever possible.
 *
 * The CPU id is stable when:
 *
 *  - IRQs are disabled;
 *  - preemption is disabled;
 *  - the task is CPU affine.
 *
 * When CONFIG_DEBUG_PREEMPT; we verify these assumption and WARN
 * when smp_processor_id() is used when the CPU id is not stable.
 */

/*
 * Allow the architecture to differentiate between a stable and unstable read.
 * For example, x86 uses an IRQ-safe asm-volatile read for the unstable but a
 * regular asm read for the stable.
 */
#ifndef __smp_processor_id
#define __smp_processor_id() raw_smp_processor_id()
#endif

#ifdef CONFIG_DEBUG_PREEMPT
  extern unsigned int debug_smp_processor_id(void);
# define smp_processor_id() debug_smp_processor_id()
#else
# define smp_processor_id() __smp_processor_id()
#endif

#define get_cpu()                ({ preempt_disable(); __smp_processor_id(); })
#define put_cpu()                preempt_enable()

/*
 * Callback to arch code if there's nosmp or maxcpus=0 on the
 * boot command line:
 */
extern void arch_disable_smp_support(void);

extern void arch_thaw_secondary_cpus_begin(void);
extern void arch_thaw_secondary_cpus_end(void);

void smp_setup_processor_id(void);

int smp_call_on_cpu(unsigned int cpu, int (*func)(void *), void *par,
                    bool phys);

/* SMP core functions */
int smpcfd_prepare_cpu(unsigned int cpu);
int smpcfd_dead_cpu(unsigned int cpu);
int smpcfd_dying_cpu(unsigned int cpu);

#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
bool csd_lock_is_stuck(void);
#else
static inline bool csd_lock_is_stuck(void) { return false; }
#endif

#endif /* __LINUX_SMP_H */



































































































    8 











 1293 
 1294 










    7 
    8 










   15 























   16 





























   11 














   12 























































    9 













    3 






































    5 








 1213 







































    6 

































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
// SPDX-License-Identifier: GPL-2.0
/*
 *  hrtimers - High-resolution kernel timers
 *
 *   Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de>
 *   Copyright(C) 2005, Red Hat, Inc., Ingo Molnar
 *
 *  data type definitions, declarations, prototypes
 *
 *  Started by: Thomas Gleixner and Ingo Molnar
 */
#ifndef _LINUX_HRTIMER_H
#define _LINUX_HRTIMER_H

#include <linux/hrtimer_defs.h>
#include <linux/hrtimer_types.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/percpu-defs.h>
#include <linux/rbtree.h>
#include <linux/timer.h>

/*
 * Mode arguments of xxx_hrtimer functions:
 *
 * HRTIMER_MODE_ABS                - Time value is absolute
 * HRTIMER_MODE_REL                - Time value is relative to now
 * HRTIMER_MODE_PINNED                - Timer is bound to CPU (is only considered
 *                                  when starting the timer)
 * HRTIMER_MODE_SOFT                - Timer callback function will be executed in
 *                                  soft irq context
 * HRTIMER_MODE_HARD                - Timer callback function will be executed in
 *                                  hard irq context even on PREEMPT_RT.
 */
enum hrtimer_mode {
        HRTIMER_MODE_ABS        = 0x00,
        HRTIMER_MODE_REL        = 0x01,
        HRTIMER_MODE_PINNED        = 0x02,
        HRTIMER_MODE_SOFT        = 0x04,
        HRTIMER_MODE_HARD        = 0x08,

        HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
        HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,

        HRTIMER_MODE_ABS_SOFT        = HRTIMER_MODE_ABS | HRTIMER_MODE_SOFT,
        HRTIMER_MODE_REL_SOFT        = HRTIMER_MODE_REL | HRTIMER_MODE_SOFT,

        HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
        HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,

        HRTIMER_MODE_ABS_HARD        = HRTIMER_MODE_ABS | HRTIMER_MODE_HARD,
        HRTIMER_MODE_REL_HARD        = HRTIMER_MODE_REL | HRTIMER_MODE_HARD,

        HRTIMER_MODE_ABS_PINNED_HARD = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_HARD,
        HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD,
};

/*
 * Values to track state of the timer
 *
 * Possible states:
 *
 * 0x00                inactive
 * 0x01                enqueued into rbtree
 *
 * The callback state is not part of the timer->state because clearing it would
 * mean touching the timer after the callback, this makes it impossible to free
 * the timer from the callback function.
 *
 * Therefore we track the callback state in:
 *
 *        timer->base->cpu_base->running == timer
 *
 * On SMP it is possible to have a "callback function running and enqueued"
 * status. It happens for example when a posix timer expired and the callback
 * queued a signal. Between dropping the lock which protects the posix timer
 * and reacquiring the base lock of the hrtimer, another CPU can deliver the
 * signal and rearm the timer.
 *
 * All state transitions are protected by cpu_base->lock.
 */
#define HRTIMER_STATE_INACTIVE        0x00
#define HRTIMER_STATE_ENQUEUED        0x01

/**
 * struct hrtimer_sleeper - simple sleeper structure
 * @timer:        embedded timer structure
 * @task:        task to wake up
 *
 * task is set to NULL, when the timer expires.
 */
struct hrtimer_sleeper {
        struct hrtimer timer;
        struct task_struct *task;
};

static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
{
        timer->node.expires = time;
        timer->_softexpires = time;
}

static inline void hrtimer_set_expires_range(struct hrtimer *timer, ktime_t time, ktime_t delta)
{
        timer->_softexpires = time;
        timer->node.expires = ktime_add_safe(time, delta);
}

static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, u64 delta)
{
        timer->_softexpires = time;
        timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta));
}

static inline void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64)
{
        timer->node.expires = tv64;
        timer->_softexpires = tv64;
}

static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time)
{
        timer->node.expires = ktime_add_safe(timer->node.expires, time);
        timer->_softexpires = ktime_add_safe(timer->_softexpires, time);
}

static inline void hrtimer_add_expires_ns(struct hrtimer *timer, u64 ns)
{
        timer->node.expires = ktime_add_ns(timer->node.expires, ns);
        timer->_softexpires = ktime_add_ns(timer->_softexpires, ns);
}

static inline ktime_t hrtimer_get_expires(const struct hrtimer *timer)
{
        return timer->node.expires;
}

static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer)
{
        return timer->_softexpires;
}

static inline s64 hrtimer_get_expires_tv64(const struct hrtimer *timer)
{
        return timer->node.expires;
}
static inline s64 hrtimer_get_softexpires_tv64(const struct hrtimer *timer)
{
        return timer->_softexpires;
}

static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
{
        return ktime_to_ns(timer->node.expires);
}

static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
{
        return ktime_sub(timer->node.expires, timer->base->get_time());
}

static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
{
        return timer->base->get_time();
}

static inline int hrtimer_is_hres_active(struct hrtimer *timer)
{
        return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
                timer->base->cpu_base->hres_active : 0;
}

#ifdef CONFIG_HIGH_RES_TIMERS
struct clock_event_device;

extern void hrtimer_interrupt(struct clock_event_device *dev);

extern unsigned int hrtimer_resolution;

#else

#define hrtimer_resolution        (unsigned int)LOW_RES_NSEC

#endif

static inline ktime_t
__hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)
{
        ktime_t rem = ktime_sub(timer->node.expires, now);

        /*
         * Adjust relative timers for the extra we added in
         * hrtimer_start_range_ns() to prevent short timeouts.
         */
        if (IS_ENABLED(CONFIG_TIME_LOW_RES) && timer->is_rel)
                rem -= hrtimer_resolution;
        return rem;
}

static inline ktime_t
hrtimer_expires_remaining_adjusted(const struct hrtimer *timer)
{
        return __hrtimer_expires_remaining_adjusted(timer,
                                                    timer->base->get_time());
}

#ifdef CONFIG_TIMERFD
extern void timerfd_clock_was_set(void);
extern void timerfd_resume(void);
#else
static inline void timerfd_clock_was_set(void) { }
static inline void timerfd_resume(void) { }
#endif

DECLARE_PER_CPU(struct tick_device, tick_cpu_device);

#ifdef CONFIG_PREEMPT_RT
void hrtimer_cancel_wait_running(const struct hrtimer *timer);
#else
static inline void hrtimer_cancel_wait_running(struct hrtimer *timer)
{
        cpu_relax();
}
#endif

/* Exported timer functions: */

/* Initialize timers: */
extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
                         enum hrtimer_mode mode);
extern void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *),
                          clockid_t clock_id, enum hrtimer_mode mode);
extern void hrtimer_setup_on_stack(struct hrtimer *timer,
                                   enum hrtimer_restart (*function)(struct hrtimer *),
                                   clockid_t clock_id, enum hrtimer_mode mode);
extern void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl, clockid_t clock_id,
                                           enum hrtimer_mode mode);

#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
#else
static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
#endif

/* Basic timer operations: */
extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
                                   u64 range_ns, const enum hrtimer_mode mode);

/**
 * hrtimer_start - (re)start an hrtimer
 * @timer:        the timer to be added
 * @tim:        expiry time
 * @mode:        timer mode: absolute (HRTIMER_MODE_ABS) or
 *                relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
 *                softirq based mode is considered for debug purpose only!
 */
static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim,
                                 const enum hrtimer_mode mode)
{
        hrtimer_start_range_ns(timer, tim, 0, mode);
}

extern int hrtimer_cancel(struct hrtimer *timer);
extern int hrtimer_try_to_cancel(struct hrtimer *timer);

static inline void hrtimer_start_expires(struct hrtimer *timer,
                                         enum hrtimer_mode mode)
{
        u64 delta;
        ktime_t soft, hard;
        soft = hrtimer_get_softexpires(timer);
        hard = hrtimer_get_expires(timer);
        delta = ktime_to_ns(ktime_sub(hard, soft));
        hrtimer_start_range_ns(timer, soft, delta, mode);
}

void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
                                   enum hrtimer_mode mode);

static inline void hrtimer_restart(struct hrtimer *timer)
{
        hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
}

/* Query timers: */
extern ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust);

/**
 * hrtimer_get_remaining - get remaining time for the timer
 * @timer:        the timer to read
 */
static inline ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
{
        return __hrtimer_get_remaining(timer, false);
}

extern u64 hrtimer_get_next_event(void);
extern u64 hrtimer_next_event_without(const struct hrtimer *exclude);

extern bool hrtimer_active(const struct hrtimer *timer);

/**
 * hrtimer_is_queued - check, whether the timer is on one of the queues
 * @timer:        Timer to check
 *
 * Returns: True if the timer is queued, false otherwise
 *
 * The function can be used lockless, but it gives only a current snapshot.
 */
static inline bool hrtimer_is_queued(struct hrtimer *timer)
{
        /* The READ_ONCE pairs with the update functions of timer->state */
        return !!(READ_ONCE(timer->state) & HRTIMER_STATE_ENQUEUED);
}

/*
 * Helper function to check, whether the timer is running the callback
 * function
 */
static inline int hrtimer_callback_running(struct hrtimer *timer)
{
        return timer->base->running == timer;
}

/**
 * hrtimer_update_function - Update the timer's callback function
 * @timer:        Timer to update
 * @function:        New callback function
 *
 * Only safe to call if the timer is not enqueued. Can be called in the callback function if the
 * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED).
 */
static inline void hrtimer_update_function(struct hrtimer *timer,
                                           enum hrtimer_restart (*function)(struct hrtimer *))
{
        guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock);

        if (WARN_ON_ONCE(hrtimer_is_queued(timer)))
                return;

        if (WARN_ON_ONCE(!function))
                return;

        timer->function = function;
}

/* Forward a hrtimer so it expires after now: */
extern u64
hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);

/**
 * hrtimer_forward_now() - forward the timer expiry so it expires after now
 * @timer:        hrtimer to forward
 * @interval:        the interval to forward
 *
 * It is a variant of hrtimer_forward(). The timer will expire after the current
 * time of the hrtimer clock base. See hrtimer_forward() for details.
 */
static inline u64 hrtimer_forward_now(struct hrtimer *timer,
                                      ktime_t interval)
{
        return hrtimer_forward(timer, timer->base->get_time(), interval);
}

/* Precise sleep: */

extern int nanosleep_copyout(struct restart_block *, struct timespec64 *);
extern long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
                              const clockid_t clockid);

extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
                                    const enum hrtimer_mode mode);
extern int schedule_hrtimeout_range_clock(ktime_t *expires,
                                          u64 delta,
                                          const enum hrtimer_mode mode,
                                          clockid_t clock_id);
extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);

/* Soft interrupt function to run the hrtimer queues: */
extern void hrtimer_run_queues(void);

/* Bootup initialization: */
extern void __init hrtimers_init(void);

/* Show pending timers: */
extern void sysrq_timer_list_show(void);

int hrtimers_prepare_cpu(unsigned int cpu);
#ifdef CONFIG_HOTPLUG_CPU
int hrtimers_cpu_dying(unsigned int cpu);
#else
#define hrtimers_cpu_dying        NULL
#endif

#endif



















































































































































































































































































































































































































































































































































































   68 




   68 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/ip.h>
#include <linux/sctp.h>
#include <net/ip.h>
#include <net/ip6_checksum.h>
#include <linux/netfilter.h>
#include <linux/netfilter_ipv4.h>
#include <net/sctp/checksum.h>
#include <net/ip_vs.h>

static int
sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp);

static int
sctp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb,
                   struct ip_vs_proto_data *pd,
                   int *verdict, struct ip_vs_conn **cpp,
                   struct ip_vs_iphdr *iph)
{
        struct ip_vs_service *svc;
        struct sctp_chunkhdr _schunkh, *sch;
        struct sctphdr *sh, _sctph;
        __be16 _ports[2], *ports = NULL;

        if (likely(!ip_vs_iph_icmp(iph))) {
                sh = skb_header_pointer(skb, iph->len, sizeof(_sctph), &_sctph);
                if (sh) {
                        sch = skb_header_pointer(skb, iph->len + sizeof(_sctph),
                                                 sizeof(_schunkh), &_schunkh);
                        if (sch) {
                                if (sch->type == SCTP_CID_ABORT ||
                                    !(sysctl_sloppy_sctp(ipvs) ||
                                      sch->type == SCTP_CID_INIT))
                                        return 1;
                                ports = &sh->source;
                        }
                }
        } else {
                ports = skb_header_pointer(
                        skb, iph->len, sizeof(_ports), &_ports);
        }

        if (!ports) {
                *verdict = NF_DROP;
                return 0;
        }

        if (likely(!ip_vs_iph_inverse(iph)))
                svc = ip_vs_service_find(ipvs, af, skb->mark, iph->protocol,
                                         &iph->daddr, ports[1]);
        else
                svc = ip_vs_service_find(ipvs, af, skb->mark, iph->protocol,
                                         &iph->saddr, ports[0]);
        if (svc) {
                int ignored;

                if (ip_vs_todrop(ipvs)) {
                        /*
                         * It seems that we are very loaded.
                         * We have to drop this packet :(
                         */
                        *verdict = NF_DROP;
                        return 0;
                }
                /*
                 * Let the virtual server select a real server for the
                 * incoming connection, and create a connection entry.
                 */
                *cpp = ip_vs_schedule(svc, skb, pd, &ignored, iph);
                if (!*cpp && ignored <= 0) {
                        if (!ignored)
                                *verdict = ip_vs_leave(svc, skb, pd, iph);
                        else
                                *verdict = NF_DROP;
                        return 0;
                }
        }
        /* NF_ACCEPT */
        return 1;
}

static void sctp_nat_csum(struct sk_buff *skb, struct sctphdr *sctph,
                          unsigned int sctphoff)
{
        sctph->checksum = sctp_compute_cksum(skb, sctphoff);
        skb->ip_summed = CHECKSUM_UNNECESSARY;
}

static int
sctp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp,
                  struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)
{
        struct sctphdr *sctph;
        unsigned int sctphoff = iph->len;
        bool payload_csum = false;

#ifdef CONFIG_IP_VS_IPV6
        if (cp->af == AF_INET6 && iph->fragoffs)
                return 1;
#endif

        /* csum_check requires unshared skb */
        if (skb_ensure_writable(skb, sctphoff + sizeof(*sctph)))
                return 0;

        if (unlikely(cp->app != NULL)) {
                int ret;

                /* Some checks before mangling */
                if (!sctp_csum_check(cp->af, skb, pp))
                        return 0;

                /* Call application helper if needed */
                ret = ip_vs_app_pkt_out(cp, skb, iph);
                if (ret == 0)
                        return 0;
                /* ret=2: csum update is needed after payload mangling */
                if (ret == 2)
                        payload_csum = true;
        }

        sctph = (void *) skb_network_header(skb) + sctphoff;

        /* Only update csum if we really have to */
        if (sctph->source != cp->vport || payload_csum ||
            skb->ip_summed == CHECKSUM_PARTIAL) {
                sctph->source = cp->vport;
                if (!skb_is_gso(skb))
                        sctp_nat_csum(skb, sctph, sctphoff);
        } else {
                skb->ip_summed = CHECKSUM_UNNECESSARY;
        }

        return 1;
}

static int
sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp,
                  struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)
{
        struct sctphdr *sctph;
        unsigned int sctphoff = iph->len;
        bool payload_csum = false;

#ifdef CONFIG_IP_VS_IPV6
        if (cp->af == AF_INET6 && iph->fragoffs)
                return 1;
#endif

        /* csum_check requires unshared skb */
        if (skb_ensure_writable(skb, sctphoff + sizeof(*sctph)))
                return 0;

        if (unlikely(cp->app != NULL)) {
                int ret;

                /* Some checks before mangling */
                if (!sctp_csum_check(cp->af, skb, pp))
                        return 0;

                /* Call application helper if needed */
                ret = ip_vs_app_pkt_in(cp, skb, iph);
                if (ret == 0)
                        return 0;
                /* ret=2: csum update is needed after payload mangling */
                if (ret == 2)
                        payload_csum = true;
        }

        sctph = (void *) skb_network_header(skb) + sctphoff;

        /* Only update csum if we really have to */
        if (sctph->dest != cp->dport || payload_csum ||
            (skb->ip_summed == CHECKSUM_PARTIAL &&
             !(skb_dst(skb)->dev->features & NETIF_F_SCTP_CRC))) {
                sctph->dest = cp->dport;
                if (!skb_is_gso(skb))
                        sctp_nat_csum(skb, sctph, sctphoff);
        } else if (skb->ip_summed != CHECKSUM_PARTIAL) {
                skb->ip_summed = CHECKSUM_UNNECESSARY;
        }

        return 1;
}

static int
sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp)
{
        unsigned int sctphoff;
        struct sctphdr *sh;
        __le32 cmp, val;

#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                sctphoff = sizeof(struct ipv6hdr);
        else
#endif
                sctphoff = ip_hdrlen(skb);

        sh = (struct sctphdr *)(skb->data + sctphoff);
        cmp = sh->checksum;
        val = sctp_compute_cksum(skb, sctphoff);

        if (val != cmp) {
                /* CRC failure, dump it. */
                IP_VS_DBG_RL_PKT(0, af, pp, skb, 0,
                                "Failed checksum for");
                return 0;
        }
        return 1;
}

enum ipvs_sctp_event_t {
        IP_VS_SCTP_DATA = 0,                /* DATA, SACK, HEARTBEATs */
        IP_VS_SCTP_INIT,
        IP_VS_SCTP_INIT_ACK,
        IP_VS_SCTP_COOKIE_ECHO,
        IP_VS_SCTP_COOKIE_ACK,
        IP_VS_SCTP_SHUTDOWN,
        IP_VS_SCTP_SHUTDOWN_ACK,
        IP_VS_SCTP_SHUTDOWN_COMPLETE,
        IP_VS_SCTP_ERROR,
        IP_VS_SCTP_ABORT,
        IP_VS_SCTP_EVENT_LAST
};

/* RFC 2960, 3.2 Chunk Field Descriptions */
static __u8 sctp_events[] = {
        [SCTP_CID_DATA]                        = IP_VS_SCTP_DATA,
        [SCTP_CID_INIT]                        = IP_VS_SCTP_INIT,
        [SCTP_CID_INIT_ACK]                = IP_VS_SCTP_INIT_ACK,
        [SCTP_CID_SACK]                        = IP_VS_SCTP_DATA,
        [SCTP_CID_HEARTBEAT]                = IP_VS_SCTP_DATA,
        [SCTP_CID_HEARTBEAT_ACK]        = IP_VS_SCTP_DATA,
        [SCTP_CID_ABORT]                = IP_VS_SCTP_ABORT,
        [SCTP_CID_SHUTDOWN]                = IP_VS_SCTP_SHUTDOWN,
        [SCTP_CID_SHUTDOWN_ACK]                = IP_VS_SCTP_SHUTDOWN_ACK,
        [SCTP_CID_ERROR]                = IP_VS_SCTP_ERROR,
        [SCTP_CID_COOKIE_ECHO]                = IP_VS_SCTP_COOKIE_ECHO,
        [SCTP_CID_COOKIE_ACK]                = IP_VS_SCTP_COOKIE_ACK,
        [SCTP_CID_ECN_ECNE]                = IP_VS_SCTP_DATA,
        [SCTP_CID_ECN_CWR]                = IP_VS_SCTP_DATA,
        [SCTP_CID_SHUTDOWN_COMPLETE]        = IP_VS_SCTP_SHUTDOWN_COMPLETE,
};

/* SCTP States:
 * See RFC 2960, 4. SCTP Association State Diagram
 *
 * New states (not in diagram):
 * - INIT1 state: use shorter timeout for dropped INIT packets
 * - REJECTED state: use shorter timeout if INIT is rejected with ABORT
 * - INIT, COOKIE_SENT, COOKIE_REPLIED, COOKIE states: for better debugging
 *
 * The states are as seen in real server. In the diagram, INIT1, INIT,
 * COOKIE_SENT and COOKIE_REPLIED processing happens in CLOSED state.
 *
 * States as per packets from client (C) and server (S):
 *
 * Setup of client connection:
 * IP_VS_SCTP_S_INIT1: First C:INIT sent, wait for S:INIT-ACK
 * IP_VS_SCTP_S_INIT: Next C:INIT sent, wait for S:INIT-ACK
 * IP_VS_SCTP_S_COOKIE_SENT: S:INIT-ACK sent, wait for C:COOKIE-ECHO
 * IP_VS_SCTP_S_COOKIE_REPLIED: C:COOKIE-ECHO sent, wait for S:COOKIE-ACK
 *
 * Setup of server connection:
 * IP_VS_SCTP_S_COOKIE_WAIT: S:INIT sent, wait for C:INIT-ACK
 * IP_VS_SCTP_S_COOKIE: C:INIT-ACK sent, wait for S:COOKIE-ECHO
 * IP_VS_SCTP_S_COOKIE_ECHOED: S:COOKIE-ECHO sent, wait for C:COOKIE-ACK
 */

#define sNO IP_VS_SCTP_S_NONE
#define sI1 IP_VS_SCTP_S_INIT1
#define sIN IP_VS_SCTP_S_INIT
#define sCS IP_VS_SCTP_S_COOKIE_SENT
#define sCR IP_VS_SCTP_S_COOKIE_REPLIED
#define sCW IP_VS_SCTP_S_COOKIE_WAIT
#define sCO IP_VS_SCTP_S_COOKIE
#define sCE IP_VS_SCTP_S_COOKIE_ECHOED
#define sES IP_VS_SCTP_S_ESTABLISHED
#define sSS IP_VS_SCTP_S_SHUTDOWN_SENT
#define sSR IP_VS_SCTP_S_SHUTDOWN_RECEIVED
#define sSA IP_VS_SCTP_S_SHUTDOWN_ACK_SENT
#define sRJ IP_VS_SCTP_S_REJECTED
#define sCL IP_VS_SCTP_S_CLOSED

static const __u8 sctp_states
        [IP_VS_DIR_LAST][IP_VS_SCTP_EVENT_LAST][IP_VS_SCTP_S_LAST] = {
        { /* INPUT */
/*        sNO, sI1, sIN, sCS, sCR, sCW, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL*/
/* d   */{sES, sI1, sIN, sCS, sCR, sCW, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* i   */{sI1, sIN, sIN, sCS, sCR, sCW, sCO, sCE, sES, sSS, sSR, sSA, sIN, sIN},
/* i_a */{sCW, sCW, sCW, sCS, sCR, sCO, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* c_e */{sCR, sIN, sIN, sCR, sCR, sCW, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* c_a */{sES, sI1, sIN, sCS, sCR, sCW, sCO, sES, sES, sSS, sSR, sSA, sRJ, sCL},
/* s   */{sSR, sI1, sIN, sCS, sCR, sCW, sCO, sCE, sSR, sSS, sSR, sSA, sRJ, sCL},
/* s_a */{sCL, sIN, sIN, sCS, sCR, sCW, sCO, sCE, sES, sCL, sSR, sCL, sRJ, sCL},
/* s_c */{sCL, sCL, sCL, sCS, sCR, sCW, sCO, sCE, sES, sSS, sSR, sCL, sRJ, sCL},
/* err */{sCL, sI1, sIN, sCS, sCR, sCW, sCO, sCL, sES, sSS, sSR, sSA, sRJ, sCL},
/* ab  */{sCL, sCL, sCL, sCL, sCL, sRJ, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL},
        },
        { /* OUTPUT */
/*        sNO, sI1, sIN, sCS, sCR, sCW, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL*/
/* d   */{sES, sI1, sIN, sCS, sCR, sCW, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* i   */{sCW, sCW, sCW, sCW, sCW, sCW, sCW, sCW, sES, sCW, sCW, sCW, sCW, sCW},
/* i_a */{sCS, sCS, sCS, sCS, sCR, sCW, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* c_e */{sCE, sCE, sCE, sCE, sCE, sCE, sCE, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* c_a */{sES, sES, sES, sES, sES, sES, sES, sES, sES, sSS, sSR, sSA, sRJ, sCL},
/* s   */{sSS, sSS, sSS, sSS, sSS, sSS, sSS, sSS, sSS, sSS, sSR, sSA, sRJ, sCL},
/* s_a */{sSA, sSA, sSA, sSA, sSA, sCW, sCO, sCE, sES, sSA, sSA, sSA, sRJ, sCL},
/* s_c */{sCL, sI1, sIN, sCS, sCR, sCW, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* err */{sCL, sCL, sCL, sCL, sCL, sCW, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* ab  */{sCL, sRJ, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL},
        },
        { /* INPUT-ONLY */
/*        sNO, sI1, sIN, sCS, sCR, sCW, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL*/
/* d   */{sES, sI1, sIN, sCS, sCR, sES, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* i   */{sI1, sIN, sIN, sIN, sIN, sIN, sCO, sCE, sES, sSS, sSR, sSA, sIN, sIN},
/* i_a */{sCE, sCE, sCE, sCE, sCE, sCE, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* c_e */{sES, sES, sES, sES, sES, sES, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* c_a */{sES, sI1, sIN, sES, sES, sCW, sES, sES, sES, sSS, sSR, sSA, sRJ, sCL},
/* s   */{sSR, sI1, sIN, sCS, sCR, sCW, sCO, sCE, sSR, sSS, sSR, sSA, sRJ, sCL},
/* s_a */{sCL, sIN, sIN, sCS, sCR, sCW, sCO, sCE, sCL, sCL, sSR, sCL, sRJ, sCL},
/* s_c */{sCL, sCL, sCL, sCL, sCL, sCW, sCO, sCE, sES, sSS, sCL, sCL, sRJ, sCL},
/* err */{sCL, sI1, sIN, sCS, sCR, sCW, sCO, sCE, sES, sSS, sSR, sSA, sRJ, sCL},
/* ab  */{sCL, sCL, sCL, sCL, sCL, sRJ, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL},
        },
};

#define IP_VS_SCTP_MAX_RTO        ((60 + 1) * HZ)

/* Timeout table[state] */
static const int sctp_timeouts[IP_VS_SCTP_S_LAST + 1] = {
        [IP_VS_SCTP_S_NONE]                        = 2 * HZ,
        [IP_VS_SCTP_S_INIT1]                        = (0 + 3 + 1) * HZ,
        [IP_VS_SCTP_S_INIT]                        = IP_VS_SCTP_MAX_RTO,
        [IP_VS_SCTP_S_COOKIE_SENT]                = IP_VS_SCTP_MAX_RTO,
        [IP_VS_SCTP_S_COOKIE_REPLIED]                = IP_VS_SCTP_MAX_RTO,
        [IP_VS_SCTP_S_COOKIE_WAIT]                = IP_VS_SCTP_MAX_RTO,
        [IP_VS_SCTP_S_COOKIE]                        = IP_VS_SCTP_MAX_RTO,
        [IP_VS_SCTP_S_COOKIE_ECHOED]                = IP_VS_SCTP_MAX_RTO,
        [IP_VS_SCTP_S_ESTABLISHED]                = 15 * 60 * HZ,
        [IP_VS_SCTP_S_SHUTDOWN_SENT]                = IP_VS_SCTP_MAX_RTO,
        [IP_VS_SCTP_S_SHUTDOWN_RECEIVED]        = IP_VS_SCTP_MAX_RTO,
        [IP_VS_SCTP_S_SHUTDOWN_ACK_SENT]        = IP_VS_SCTP_MAX_RTO,
        [IP_VS_SCTP_S_REJECTED]                        = (0 + 3 + 1) * HZ,
        [IP_VS_SCTP_S_CLOSED]                        = IP_VS_SCTP_MAX_RTO,
        [IP_VS_SCTP_S_LAST]                        = 2 * HZ,
};

static const char *sctp_state_name_table[IP_VS_SCTP_S_LAST + 1] = {
        [IP_VS_SCTP_S_NONE]                        = "NONE",
        [IP_VS_SCTP_S_INIT1]                        = "INIT1",
        [IP_VS_SCTP_S_INIT]                        = "INIT",
        [IP_VS_SCTP_S_COOKIE_SENT]                = "C-SENT",
        [IP_VS_SCTP_S_COOKIE_REPLIED]                = "C-REPLIED",
        [IP_VS_SCTP_S_COOKIE_WAIT]                = "C-WAIT",
        [IP_VS_SCTP_S_COOKIE]                        = "COOKIE",
        [IP_VS_SCTP_S_COOKIE_ECHOED]                = "C-ECHOED",
        [IP_VS_SCTP_S_ESTABLISHED]                = "ESTABLISHED",
        [IP_VS_SCTP_S_SHUTDOWN_SENT]                = "S-SENT",
        [IP_VS_SCTP_S_SHUTDOWN_RECEIVED]        = "S-RECEIVED",
        [IP_VS_SCTP_S_SHUTDOWN_ACK_SENT]        = "S-ACK-SENT",
        [IP_VS_SCTP_S_REJECTED]                        = "REJECTED",
        [IP_VS_SCTP_S_CLOSED]                        = "CLOSED",
        [IP_VS_SCTP_S_LAST]                        = "BUG!",
};


static const char *sctp_state_name(int state)
{
        if (state >= IP_VS_SCTP_S_LAST)
                return "ERR!";
        if (sctp_state_name_table[state])
                return sctp_state_name_table[state];
        return "?";
}

static inline void
set_sctp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp,
                int direction, const struct sk_buff *skb)
{
        struct sctp_chunkhdr _sctpch, *sch;
        unsigned char chunk_type;
        int event, next_state;
        int ihl, cofs;

#ifdef CONFIG_IP_VS_IPV6
        ihl = cp->af == AF_INET ? ip_hdrlen(skb) : sizeof(struct ipv6hdr);
#else
        ihl = ip_hdrlen(skb);
#endif

        cofs = ihl + sizeof(struct sctphdr);
        sch = skb_header_pointer(skb, cofs, sizeof(_sctpch), &_sctpch);
        if (sch == NULL)
                return;

        chunk_type = sch->type;
        /*
         * Section 3: Multiple chunks can be bundled into one SCTP packet
         * up to the MTU size, except for the INIT, INIT ACK, and
         * SHUTDOWN COMPLETE chunks. These chunks MUST NOT be bundled with
         * any other chunk in a packet.
         *
         * Section 3.3.7: DATA chunks MUST NOT be bundled with ABORT. Control
         * chunks (except for INIT, INIT ACK, and SHUTDOWN COMPLETE) MAY be
         * bundled with an ABORT, but they MUST be placed before the ABORT
         * in the SCTP packet or they will be ignored by the receiver.
         */
        if ((sch->type == SCTP_CID_COOKIE_ECHO) ||
            (sch->type == SCTP_CID_COOKIE_ACK)) {
                int clen = ntohs(sch->length);

                if (clen >= sizeof(_sctpch)) {
                        sch = skb_header_pointer(skb, cofs + ALIGN(clen, 4),
                                                 sizeof(_sctpch), &_sctpch);
                        if (sch && sch->type == SCTP_CID_ABORT)
                                chunk_type = sch->type;
                }
        }

        event = (chunk_type < sizeof(sctp_events)) ?
                sctp_events[chunk_type] : IP_VS_SCTP_DATA;

        /* Update direction to INPUT_ONLY if necessary
         * or delete NO_OUTPUT flag if output packet detected
         */
        if (cp->flags & IP_VS_CONN_F_NOOUTPUT) {
                if (direction == IP_VS_DIR_OUTPUT)
                        cp->flags &= ~IP_VS_CONN_F_NOOUTPUT;
                else
                        direction = IP_VS_DIR_INPUT_ONLY;
        }

        next_state = sctp_states[direction][event][cp->state];

        if (next_state != cp->state) {
                struct ip_vs_dest *dest = cp->dest;

                IP_VS_DBG_BUF(8, "%s %s  %s:%d->"
                                "%s:%d state: %s->%s conn->refcnt:%d\n",
                                pd->pp->name,
                                ((direction == IP_VS_DIR_OUTPUT) ?
                                 "output " : "input "),
                                IP_VS_DBG_ADDR(cp->daf, &cp->daddr),
                                ntohs(cp->dport),
                                IP_VS_DBG_ADDR(cp->af, &cp->caddr),
                                ntohs(cp->cport),
                                sctp_state_name(cp->state),
                                sctp_state_name(next_state),
                                refcount_read(&cp->refcnt));
                if (dest) {
                        if (!(cp->flags & IP_VS_CONN_F_INACTIVE) &&
                                (next_state != IP_VS_SCTP_S_ESTABLISHED)) {
                                atomic_dec(&dest->activeconns);
                                atomic_inc(&dest->inactconns);
                                cp->flags |= IP_VS_CONN_F_INACTIVE;
                        } else if ((cp->flags & IP_VS_CONN_F_INACTIVE) &&
                                   (next_state == IP_VS_SCTP_S_ESTABLISHED)) {
                                atomic_inc(&dest->activeconns);
                                atomic_dec(&dest->inactconns);
                                cp->flags &= ~IP_VS_CONN_F_INACTIVE;
                        }
                }
                if (next_state == IP_VS_SCTP_S_ESTABLISHED)
                        ip_vs_control_assure_ct(cp);
        }
        if (likely(pd))
                cp->timeout = pd->timeout_table[cp->state = next_state];
        else        /* What to do ? */
                cp->timeout = sctp_timeouts[cp->state = next_state];
}

static void
sctp_state_transition(struct ip_vs_conn *cp, int direction,
                const struct sk_buff *skb, struct ip_vs_proto_data *pd)
{
        spin_lock_bh(&cp->lock);
        set_sctp_state(pd, cp, direction, skb);
        spin_unlock_bh(&cp->lock);
}

static inline __u16 sctp_app_hashkey(__be16 port)
{
        return (((__force u16)port >> SCTP_APP_TAB_BITS) ^ (__force u16)port)
                & SCTP_APP_TAB_MASK;
}

static int sctp_register_app(struct netns_ipvs *ipvs, struct ip_vs_app *inc)
{
        struct ip_vs_app *i;
        __u16 hash;
        __be16 port = inc->port;
        int ret = 0;
        struct ip_vs_proto_data *pd = ip_vs_proto_data_get(ipvs, IPPROTO_SCTP);

        hash = sctp_app_hashkey(port);

        list_for_each_entry(i, &ipvs->sctp_apps[hash], p_list) {
                if (i->port == port) {
                        ret = -EEXIST;
                        goto out;
                }
        }
        list_add_rcu(&inc->p_list, &ipvs->sctp_apps[hash]);
        atomic_inc(&pd->appcnt);
out:

        return ret;
}

static void sctp_unregister_app(struct netns_ipvs *ipvs, struct ip_vs_app *inc)
{
        struct ip_vs_proto_data *pd = ip_vs_proto_data_get(ipvs, IPPROTO_SCTP);

        atomic_dec(&pd->appcnt);
        list_del_rcu(&inc->p_list);
}

static int sctp_app_conn_bind(struct ip_vs_conn *cp)
{
        struct netns_ipvs *ipvs = cp->ipvs;
        int hash;
        struct ip_vs_app *inc;
        int result = 0;

        /* Default binding: bind app only for NAT */
        if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)
                return 0;
        /* Lookup application incarnations and bind the right one */
        hash = sctp_app_hashkey(cp->vport);

        list_for_each_entry_rcu(inc, &ipvs->sctp_apps[hash], p_list) {
                if (inc->port == cp->vport) {
                        if (unlikely(!ip_vs_app_inc_get(inc)))
                                break;

                        IP_VS_DBG_BUF(9, "%s: Binding conn %s:%u->"
                                        "%s:%u to app %s on port %u\n",
                                        __func__,
                                        IP_VS_DBG_ADDR(cp->af, &cp->caddr),
                                        ntohs(cp->cport),
                                        IP_VS_DBG_ADDR(cp->af, &cp->vaddr),
                                        ntohs(cp->vport),
                                        inc->name, ntohs(inc->port));
                        cp->app = inc;
                        if (inc->init_conn)
                                result = inc->init_conn(inc, cp);
                        break;
                }
        }

        return result;
}

/* ---------------------------------------------
 *   timeouts is netns related now.
 * ---------------------------------------------
 */
static int __ip_vs_sctp_init(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd)
{
        ip_vs_init_hash_table(ipvs->sctp_apps, SCTP_APP_TAB_SIZE);
        pd->timeout_table = ip_vs_create_timeout_table((int *)sctp_timeouts,
                                                        sizeof(sctp_timeouts));
        if (!pd->timeout_table)
                return -ENOMEM;
        return 0;
}

static void __ip_vs_sctp_exit(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd)
{
        kfree(pd->timeout_table);
}

struct ip_vs_protocol ip_vs_protocol_sctp = {
        .name                = "SCTP",
        .protocol        = IPPROTO_SCTP,
        .num_states        = IP_VS_SCTP_S_LAST,
        .dont_defrag        = 0,
        .init                = NULL,
        .exit                = NULL,
        .init_netns        = __ip_vs_sctp_init,
        .exit_netns        = __ip_vs_sctp_exit,
        .register_app        = sctp_register_app,
        .unregister_app = sctp_unregister_app,
        .conn_schedule        = sctp_conn_schedule,
        .conn_in_get        = ip_vs_conn_in_get_proto,
        .conn_out_get        = ip_vs_conn_out_get_proto,
        .snat_handler        = sctp_snat_handler,
        .dnat_handler        = sctp_dnat_handler,
        .state_name        = sctp_state_name,
        .state_transition = sctp_state_transition,
        .app_conn_bind        = sctp_app_conn_bind,
        .debug_packet        = ip_vs_tcpudp_debug_packet,
        .timeout_change        = NULL,
};


















 4155 





 4152 









   45 
   12 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_PGTABLE_INVERT_H
#define _ASM_PGTABLE_INVERT_H 1

#ifndef __ASSEMBLY__

/*
 * A clear pte value is special, and doesn't get inverted.
 *
 * Note that even users that only pass a pgprot_t (rather
 * than a full pte) won't trigger the special zero case,
 * because even PAGE_NONE has _PAGE_PROTNONE | _PAGE_ACCESSED
 * set. So the all zero case really is limited to just the
 * cleared page table entry case.
 */
static inline bool __pte_needs_invert(u64 val)
{
        return val && !(val & _PAGE_PRESENT);
}

/* Get a mask to xor with the page table entry to get the correct pfn. */
static inline u64 protnone_mask(u64 val)
{
        return __pte_needs_invert(val) ?  ~0ull : 0;
}

static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask)
{
        /*
         * When a PTE transitions from NONE to !NONE or vice-versa
         * invert the PFN part to stop speculation.
         * pte_pfn undoes this when needed.
         */
        if (__pte_needs_invert(oldval) != __pte_needs_invert(val))
                val = (val & ~mask) | (~val & mask);
        return val;
}

#endif /* __ASSEMBLY__ */

#endif























   32 
   32 

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
// SPDX-License-Identifier: GPL-2.0
#include <linux/bitops.h>
#include <linux/bug.h>
#include <linux/export.h>
#include <linux/limits.h>
#include <linux/math.h>
#include <linux/minmax.h>
#include <linux/types.h>

#include <linux/reciprocal_div.h>

/*
 * For a description of the algorithm please have a look at
 * include/linux/reciprocal_div.h
 */

struct reciprocal_value reciprocal_value(u32 d)
{
        struct reciprocal_value R;
        u64 m;
        int l;

        l = fls(d - 1);
        m = ((1ULL << 32) * ((1ULL << l) - d));
        do_div(m, d);
        ++m;
        R.m = (u32)m;
        R.sh1 = min(l, 1);
        R.sh2 = max(l - 1, 0);

        return R;
}
EXPORT_SYMBOL(reciprocal_value);

struct reciprocal_value_adv reciprocal_value_adv(u32 d, u8 prec)
{
        struct reciprocal_value_adv R;
        u32 l, post_shift;
        u64 mhigh, mlow;

        /* ceil(log2(d)) */
        l = fls(d - 1);
        /* NOTE: mlow/mhigh could overflow u64 when l == 32. This case needs to
         * be handled before calling "reciprocal_value_adv", please see the
         * comment at include/linux/reciprocal_div.h.
         */
        WARN(l == 32,
             "ceil(log2(0x%08x)) == 32, %s doesn't support such divisor",
             d, __func__);
        post_shift = l;
        mlow = 1ULL << (32 + l);
        do_div(mlow, d);
        mhigh = (1ULL << (32 + l)) + (1ULL << (32 + l - prec));
        do_div(mhigh, d);

        for (; post_shift > 0; post_shift--) {
                u64 lo = mlow >> 1, hi = mhigh >> 1;

                if (lo >= hi)
                        break;

                mlow = lo;
                mhigh = hi;
        }

        R.m = (u32)mhigh;
        R.sh = post_shift;
        R.exp = l;
        R.is_wide_m = mhigh > U32_MAX;

        return R;
}
EXPORT_SYMBOL(reciprocal_value_adv);






















































































































































































































































































































































































































    4 








































    4 









































































    4 



   68 




   68 


   68 

   68 

   68 




   68 

   68 













































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
// SPDX-License-Identifier: GPL-2.0-only
/*
 * count the number of connections matching an arbitrary key.
 *
 * (C) 2017 Red Hat GmbH
 * Author: Florian Westphal <fw@strlen.de>
 *
 * split from xt_connlimit.c:
 *   (c) 2000 Gerd Knorr <kraxel@bytesex.org>
 *   Nov 2002: Martin Bene <martin.bene@icomedias.com>:
 *                only ignore TIME_WAIT or gone connections
 *   (C) CC Computer Consultants GmbH, 2007
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/jhash.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/rbtree.h>
#include <linux/module.h>
#include <linux/random.h>
#include <linux/skbuff.h>
#include <linux/spinlock.h>
#include <linux/netfilter/nf_conntrack_tcp.h>
#include <linux/netfilter/x_tables.h>
#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_count.h>
#include <net/netfilter/nf_conntrack_core.h>
#include <net/netfilter/nf_conntrack_tuple.h>
#include <net/netfilter/nf_conntrack_zones.h>

#define CONNCOUNT_SLOTS                256U

#define CONNCOUNT_GC_MAX_NODES        8
#define MAX_KEYLEN                5

/* we will save the tuples of all connections we care about */
struct nf_conncount_tuple {
        struct list_head                node;
        struct nf_conntrack_tuple        tuple;
        struct nf_conntrack_zone        zone;
        int                                cpu;
        u32                                jiffies32;
};

struct nf_conncount_rb {
        struct rb_node node;
        struct nf_conncount_list list;
        u32 key[MAX_KEYLEN];
        struct rcu_head rcu_head;
};

static spinlock_t nf_conncount_locks[CONNCOUNT_SLOTS] __cacheline_aligned_in_smp;

struct nf_conncount_data {
        unsigned int keylen;
        struct rb_root root[CONNCOUNT_SLOTS];
        struct net *net;
        struct work_struct gc_work;
        unsigned long pending_trees[BITS_TO_LONGS(CONNCOUNT_SLOTS)];
        unsigned int gc_tree;
};

static u_int32_t conncount_rnd __read_mostly;
static struct kmem_cache *conncount_rb_cachep __read_mostly;
static struct kmem_cache *conncount_conn_cachep __read_mostly;

static inline bool already_closed(const struct nf_conn *conn)
{
        if (nf_ct_protonum(conn) == IPPROTO_TCP)
                return conn->proto.tcp.state == TCP_CONNTRACK_TIME_WAIT ||
                       conn->proto.tcp.state == TCP_CONNTRACK_CLOSE;
        else
                return false;
}

static int key_diff(const u32 *a, const u32 *b, unsigned int klen)
{
        return memcmp(a, b, klen * sizeof(u32));
}

static void conn_free(struct nf_conncount_list *list,
                      struct nf_conncount_tuple *conn)
{
        lockdep_assert_held(&list->list_lock);

        list->count--;
        list_del(&conn->node);

        kmem_cache_free(conncount_conn_cachep, conn);
}

static const struct nf_conntrack_tuple_hash *
find_or_evict(struct net *net, struct nf_conncount_list *list,
              struct nf_conncount_tuple *conn)
{
        const struct nf_conntrack_tuple_hash *found;
        unsigned long a, b;
        int cpu = raw_smp_processor_id();
        u32 age;

        found = nf_conntrack_find_get(net, &conn->zone, &conn->tuple);
        if (found)
                return found;
        b = conn->jiffies32;
        a = (u32)jiffies;

        /* conn might have been added just before by another cpu and
         * might still be unconfirmed.  In this case, nf_conntrack_find()
         * returns no result.  Thus only evict if this cpu added the
         * stale entry or if the entry is older than two jiffies.
         */
        age = a - b;
        if (conn->cpu == cpu || age >= 2) {
                conn_free(list, conn);
                return ERR_PTR(-ENOENT);
        }

        return ERR_PTR(-EAGAIN);
}

static int __nf_conncount_add(struct net *net,
                              struct nf_conncount_list *list,
                              const struct nf_conntrack_tuple *tuple,
                              const struct nf_conntrack_zone *zone)
{
        const struct nf_conntrack_tuple_hash *found;
        struct nf_conncount_tuple *conn, *conn_n;
        struct nf_conn *found_ct;
        unsigned int collect = 0;

        if (time_is_after_eq_jiffies((unsigned long)list->last_gc))
                goto add_new_node;

        /* check the saved connections */
        list_for_each_entry_safe(conn, conn_n, &list->head, node) {
                if (collect > CONNCOUNT_GC_MAX_NODES)
                        break;

                found = find_or_evict(net, list, conn);
                if (IS_ERR(found)) {
                        /* Not found, but might be about to be confirmed */
                        if (PTR_ERR(found) == -EAGAIN) {
                                if (nf_ct_tuple_equal(&conn->tuple, tuple) &&
                                    nf_ct_zone_id(&conn->zone, conn->zone.dir) ==
                                    nf_ct_zone_id(zone, zone->dir))
                                        return 0; /* already exists */
                        } else {
                                collect++;
                        }
                        continue;
                }

                found_ct = nf_ct_tuplehash_to_ctrack(found);

                if (nf_ct_tuple_equal(&conn->tuple, tuple) &&
                    nf_ct_zone_equal(found_ct, zone, zone->dir)) {
                        /*
                         * We should not see tuples twice unless someone hooks
                         * this into a table without "-p tcp --syn".
                         *
                         * Attempt to avoid a re-add in this case.
                         */
                        nf_ct_put(found_ct);
                        return 0;
                } else if (already_closed(found_ct)) {
                        /*
                         * we do not care about connections which are
                         * closed already -> ditch it
                         */
                        nf_ct_put(found_ct);
                        conn_free(list, conn);
                        collect++;
                        continue;
                }

                nf_ct_put(found_ct);
        }

add_new_node:
        if (WARN_ON_ONCE(list->count > INT_MAX))
                return -EOVERFLOW;

        conn = kmem_cache_alloc(conncount_conn_cachep, GFP_ATOMIC);
        if (conn == NULL)
                return -ENOMEM;

        conn->tuple = *tuple;
        conn->zone = *zone;
        conn->cpu = raw_smp_processor_id();
        conn->jiffies32 = (u32)jiffies;
        list_add_tail(&conn->node, &list->head);
        list->count++;
        list->last_gc = (u32)jiffies;
        return 0;
}

int nf_conncount_add(struct net *net,
                     struct nf_conncount_list *list,
                     const struct nf_conntrack_tuple *tuple,
                     const struct nf_conntrack_zone *zone)
{
        int ret;

        /* check the saved connections */
        spin_lock_bh(&list->list_lock);
        ret = __nf_conncount_add(net, list, tuple, zone);
        spin_unlock_bh(&list->list_lock);

        return ret;
}
EXPORT_SYMBOL_GPL(nf_conncount_add);

void nf_conncount_list_init(struct nf_conncount_list *list)
{
        spin_lock_init(&list->list_lock);
        INIT_LIST_HEAD(&list->head);
        list->count = 0;
        list->last_gc = (u32)jiffies;
}
EXPORT_SYMBOL_GPL(nf_conncount_list_init);

/* Return true if the list is empty. Must be called with BH disabled. */
bool nf_conncount_gc_list(struct net *net,
                          struct nf_conncount_list *list)
{
        const struct nf_conntrack_tuple_hash *found;
        struct nf_conncount_tuple *conn, *conn_n;
        struct nf_conn *found_ct;
        unsigned int collected = 0;
        bool ret = false;

        /* don't bother if we just did GC */
        if (time_is_after_eq_jiffies((unsigned long)READ_ONCE(list->last_gc)))
                return false;

        /* don't bother if other cpu is already doing GC */
        if (!spin_trylock(&list->list_lock))
                return false;

        list_for_each_entry_safe(conn, conn_n, &list->head, node) {
                found = find_or_evict(net, list, conn);
                if (IS_ERR(found)) {
                        if (PTR_ERR(found) == -ENOENT)
                                collected++;
                        continue;
                }

                found_ct = nf_ct_tuplehash_to_ctrack(found);
                if (already_closed(found_ct)) {
                        /*
                         * we do not care about connections which are
                         * closed already -> ditch it
                         */
                        nf_ct_put(found_ct);
                        conn_free(list, conn);
                        collected++;
                        continue;
                }

                nf_ct_put(found_ct);
                if (collected > CONNCOUNT_GC_MAX_NODES)
                        break;
        }

        if (!list->count)
                ret = true;
        list->last_gc = (u32)jiffies;
        spin_unlock(&list->list_lock);

        return ret;
}
EXPORT_SYMBOL_GPL(nf_conncount_gc_list);

static void __tree_nodes_free(struct rcu_head *h)
{
        struct nf_conncount_rb *rbconn;

        rbconn = container_of(h, struct nf_conncount_rb, rcu_head);
        kmem_cache_free(conncount_rb_cachep, rbconn);
}

/* caller must hold tree nf_conncount_locks[] lock */
static void tree_nodes_free(struct rb_root *root,
                            struct nf_conncount_rb *gc_nodes[],
                            unsigned int gc_count)
{
        struct nf_conncount_rb *rbconn;

        while (gc_count) {
                rbconn = gc_nodes[--gc_count];
                spin_lock(&rbconn->list.list_lock);
                if (!rbconn->list.count) {
                        rb_erase(&rbconn->node, root);
                        call_rcu(&rbconn->rcu_head, __tree_nodes_free);
                }
                spin_unlock(&rbconn->list.list_lock);
        }
}

static void schedule_gc_worker(struct nf_conncount_data *data, int tree)
{
        set_bit(tree, data->pending_trees);
        schedule_work(&data->gc_work);
}

static unsigned int
insert_tree(struct net *net,
            struct nf_conncount_data *data,
            struct rb_root *root,
            unsigned int hash,
            const u32 *key,
            const struct nf_conntrack_tuple *tuple,
            const struct nf_conntrack_zone *zone)
{
        struct nf_conncount_rb *gc_nodes[CONNCOUNT_GC_MAX_NODES];
        struct rb_node **rbnode, *parent;
        struct nf_conncount_rb *rbconn;
        struct nf_conncount_tuple *conn;
        unsigned int count = 0, gc_count = 0;
        bool do_gc = true;

        spin_lock_bh(&nf_conncount_locks[hash]);
restart:
        parent = NULL;
        rbnode = &(root->rb_node);
        while (*rbnode) {
                int diff;
                rbconn = rb_entry(*rbnode, struct nf_conncount_rb, node);

                parent = *rbnode;
                diff = key_diff(key, rbconn->key, data->keylen);
                if (diff < 0) {
                        rbnode = &((*rbnode)->rb_left);
                } else if (diff > 0) {
                        rbnode = &((*rbnode)->rb_right);
                } else {
                        int ret;

                        ret = nf_conncount_add(net, &rbconn->list, tuple, zone);
                        if (ret)
                                count = 0; /* hotdrop */
                        else
                                count = rbconn->list.count;
                        tree_nodes_free(root, gc_nodes, gc_count);
                        goto out_unlock;
                }

                if (gc_count >= ARRAY_SIZE(gc_nodes))
                        continue;

                if (do_gc && nf_conncount_gc_list(net, &rbconn->list))
                        gc_nodes[gc_count++] = rbconn;
        }

        if (gc_count) {
                tree_nodes_free(root, gc_nodes, gc_count);
                schedule_gc_worker(data, hash);
                gc_count = 0;
                do_gc = false;
                goto restart;
        }

        /* expected case: match, insert new node */
        rbconn = kmem_cache_alloc(conncount_rb_cachep, GFP_ATOMIC);
        if (rbconn == NULL)
                goto out_unlock;

        conn = kmem_cache_alloc(conncount_conn_cachep, GFP_ATOMIC);
        if (conn == NULL) {
                kmem_cache_free(conncount_rb_cachep, rbconn);
                goto out_unlock;
        }

        conn->tuple = *tuple;
        conn->zone = *zone;
        memcpy(rbconn->key, key, sizeof(u32) * data->keylen);

        nf_conncount_list_init(&rbconn->list);
        list_add(&conn->node, &rbconn->list.head);
        count = 1;
        rbconn->list.count = count;

        rb_link_node_rcu(&rbconn->node, parent, rbnode);
        rb_insert_color(&rbconn->node, root);
out_unlock:
        spin_unlock_bh(&nf_conncount_locks[hash]);
        return count;
}

static unsigned int
count_tree(struct net *net,
           struct nf_conncount_data *data,
           const u32 *key,
           const struct nf_conntrack_tuple *tuple,
           const struct nf_conntrack_zone *zone)
{
        struct rb_root *root;
        struct rb_node *parent;
        struct nf_conncount_rb *rbconn;
        unsigned int hash;

        hash = jhash2(key, data->keylen, conncount_rnd) % CONNCOUNT_SLOTS;
        root = &data->root[hash];

        parent = rcu_dereference_raw(root->rb_node);
        while (parent) {
                int diff;

                rbconn = rb_entry(parent, struct nf_conncount_rb, node);

                diff = key_diff(key, rbconn->key, data->keylen);
                if (diff < 0) {
                        parent = rcu_dereference_raw(parent->rb_left);
                } else if (diff > 0) {
                        parent = rcu_dereference_raw(parent->rb_right);
                } else {
                        int ret;

                        if (!tuple) {
                                nf_conncount_gc_list(net, &rbconn->list);
                                return rbconn->list.count;
                        }

                        spin_lock_bh(&rbconn->list.list_lock);
                        /* Node might be about to be free'd.
                         * We need to defer to insert_tree() in this case.
                         */
                        if (rbconn->list.count == 0) {
                                spin_unlock_bh(&rbconn->list.list_lock);
                                break;
                        }

                        /* same source network -> be counted! */
                        ret = __nf_conncount_add(net, &rbconn->list, tuple, zone);
                        spin_unlock_bh(&rbconn->list.list_lock);
                        if (ret)
                                return 0; /* hotdrop */
                        else
                                return rbconn->list.count;
                }
        }

        if (!tuple)
                return 0;

        return insert_tree(net, data, root, hash, key, tuple, zone);
}

static void tree_gc_worker(struct work_struct *work)
{
        struct nf_conncount_data *data = container_of(work, struct nf_conncount_data, gc_work);
        struct nf_conncount_rb *gc_nodes[CONNCOUNT_GC_MAX_NODES], *rbconn;
        struct rb_root *root;
        struct rb_node *node;
        unsigned int tree, next_tree, gc_count = 0;

        tree = data->gc_tree % CONNCOUNT_SLOTS;
        root = &data->root[tree];

        local_bh_disable();
        rcu_read_lock();
        for (node = rb_first(root); node != NULL; node = rb_next(node)) {
                rbconn = rb_entry(node, struct nf_conncount_rb, node);
                if (nf_conncount_gc_list(data->net, &rbconn->list))
                        gc_count++;
        }
        rcu_read_unlock();
        local_bh_enable();

        cond_resched();

        spin_lock_bh(&nf_conncount_locks[tree]);
        if (gc_count < ARRAY_SIZE(gc_nodes))
                goto next; /* do not bother */

        gc_count = 0;
        node = rb_first(root);
        while (node != NULL) {
                rbconn = rb_entry(node, struct nf_conncount_rb, node);
                node = rb_next(node);

                if (rbconn->list.count > 0)
                        continue;

                gc_nodes[gc_count++] = rbconn;
                if (gc_count >= ARRAY_SIZE(gc_nodes)) {
                        tree_nodes_free(root, gc_nodes, gc_count);
                        gc_count = 0;
                }
        }

        tree_nodes_free(root, gc_nodes, gc_count);
next:
        clear_bit(tree, data->pending_trees);

        next_tree = (tree + 1) % CONNCOUNT_SLOTS;
        next_tree = find_next_bit(data->pending_trees, CONNCOUNT_SLOTS, next_tree);

        if (next_tree < CONNCOUNT_SLOTS) {
                data->gc_tree = next_tree;
                schedule_work(work);
        }

        spin_unlock_bh(&nf_conncount_locks[tree]);
}

/* Count and return number of conntrack entries in 'net' with particular 'key'.
 * If 'tuple' is not null, insert it into the accounting data structure.
 * Call with RCU read lock.
 */
unsigned int nf_conncount_count(struct net *net,
                                struct nf_conncount_data *data,
                                const u32 *key,
                                const struct nf_conntrack_tuple *tuple,
                                const struct nf_conntrack_zone *zone)
{
        return count_tree(net, data, key, tuple, zone);
}
EXPORT_SYMBOL_GPL(nf_conncount_count);

struct nf_conncount_data *nf_conncount_init(struct net *net, unsigned int keylen)
{
        struct nf_conncount_data *data;
        int i;

        if (keylen % sizeof(u32) ||
            keylen / sizeof(u32) > MAX_KEYLEN ||
            keylen == 0)
                return ERR_PTR(-EINVAL);

        net_get_random_once(&conncount_rnd, sizeof(conncount_rnd));

        data = kmalloc(sizeof(*data), GFP_KERNEL);
        if (!data)
                return ERR_PTR(-ENOMEM);

        for (i = 0; i < ARRAY_SIZE(data->root); ++i)
                data->root[i] = RB_ROOT;

        data->keylen = keylen / sizeof(u32);
        data->net = net;
        INIT_WORK(&data->gc_work, tree_gc_worker);

        return data;
}
EXPORT_SYMBOL_GPL(nf_conncount_init);

void nf_conncount_cache_free(struct nf_conncount_list *list)
{
        struct nf_conncount_tuple *conn, *conn_n;

        list_for_each_entry_safe(conn, conn_n, &list->head, node)
                kmem_cache_free(conncount_conn_cachep, conn);
}
EXPORT_SYMBOL_GPL(nf_conncount_cache_free);

static void destroy_tree(struct rb_root *r)
{
        struct nf_conncount_rb *rbconn;
        struct rb_node *node;

        while ((node = rb_first(r)) != NULL) {
                rbconn = rb_entry(node, struct nf_conncount_rb, node);

                rb_erase(node, r);

                nf_conncount_cache_free(&rbconn->list);

                kmem_cache_free(conncount_rb_cachep, rbconn);
        }
}

void nf_conncount_destroy(struct net *net, struct nf_conncount_data *data)
{
        unsigned int i;

        cancel_work_sync(&data->gc_work);

        for (i = 0; i < ARRAY_SIZE(data->root); ++i)
                destroy_tree(&data->root[i]);

        kfree(data);
}
EXPORT_SYMBOL_GPL(nf_conncount_destroy);

static int __init nf_conncount_modinit(void)
{
        int i;

        for (i = 0; i < CONNCOUNT_SLOTS; ++i)
                spin_lock_init(&nf_conncount_locks[i]);

        conncount_conn_cachep = KMEM_CACHE(nf_conncount_tuple, 0);
        if (!conncount_conn_cachep)
                return -ENOMEM;

        conncount_rb_cachep = KMEM_CACHE(nf_conncount_rb, 0);
        if (!conncount_rb_cachep) {
                kmem_cache_destroy(conncount_conn_cachep);
                return -ENOMEM;
        }

        return 0;
}

static void __exit nf_conncount_modexit(void)
{
        kmem_cache_destroy(conncount_conn_cachep);
        kmem_cache_destroy(conncount_rb_cachep);
}

module_init(nf_conncount_modinit);
module_exit(nf_conncount_modexit);
MODULE_AUTHOR("Jan Engelhardt <jengelh@medozas.de>");
MODULE_AUTHOR("Florian Westphal <fw@strlen.de>");
MODULE_DESCRIPTION("netfilter: count number of connections matching a key");
MODULE_LICENSE("GPL");












































































































































































































































































































    1 













    3 



    3 

    3 



























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
/*
 * JFFS2 -- Journalling Flash File System, Version 2.
 *
 * Copyright © 2001-2007 Red Hat, Inc.
 *
 * Created by David Woodhouse <dwmw2@infradead.org>
 *
 * For licensing information, see the file 'LICENCE' in this directory.
 *
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/fs.h>
#include <linux/err.h>
#include <linux/mount.h>
#include <linux/fs_context.h>
#include <linux/fs_parser.h>
#include <linux/jffs2.h>
#include <linux/pagemap.h>
#include <linux/mtd/super.h>
#include <linux/ctype.h>
#include <linux/namei.h>
#include <linux/seq_file.h>
#include <linux/exportfs.h>
#include "compr.h"
#include "nodelist.h"

static void jffs2_put_super(struct super_block *);

static struct kmem_cache *jffs2_inode_cachep;

static struct inode *jffs2_alloc_inode(struct super_block *sb)
{
        struct jffs2_inode_info *f;

        f = alloc_inode_sb(sb, jffs2_inode_cachep, GFP_KERNEL);
        if (!f)
                return NULL;
        return &f->vfs_inode;
}

static void jffs2_free_inode(struct inode *inode)
{
        struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode);

        kfree(f->target);
        kmem_cache_free(jffs2_inode_cachep, f);
}

static void jffs2_i_init_once(void *foo)
{
        struct jffs2_inode_info *f = foo;

        mutex_init(&f->sem);
        f->target = NULL;
        inode_init_once(&f->vfs_inode);
}

static const char *jffs2_compr_name(unsigned int compr)
{
        switch (compr) {
        case JFFS2_COMPR_MODE_NONE:
                return "none";
#ifdef CONFIG_JFFS2_LZO
        case JFFS2_COMPR_MODE_FORCELZO:
                return "lzo";
#endif
#ifdef CONFIG_JFFS2_ZLIB
        case JFFS2_COMPR_MODE_FORCEZLIB:
                return "zlib";
#endif
        default:
                /* should never happen; programmer error */
                WARN_ON(1);
                return "";
        }
}

static int jffs2_show_options(struct seq_file *s, struct dentry *root)
{
        struct jffs2_sb_info *c = JFFS2_SB_INFO(root->d_sb);
        struct jffs2_mount_opts *opts = &c->mount_opts;

        if (opts->override_compr)
                seq_printf(s, ",compr=%s", jffs2_compr_name(opts->compr));
        if (opts->set_rp_size)
                seq_printf(s, ",rp_size=%u", opts->rp_size / 1024);

        return 0;
}

static int jffs2_sync_fs(struct super_block *sb, int wait)
{
        struct jffs2_sb_info *c = JFFS2_SB_INFO(sb);

#ifdef CONFIG_JFFS2_FS_WRITEBUFFER
        if (jffs2_is_writebuffered(c))
                cancel_delayed_work_sync(&c->wbuf_dwork);
#endif

        mutex_lock(&c->alloc_sem);
        jffs2_flush_wbuf_pad(c);
        mutex_unlock(&c->alloc_sem);
        return 0;
}

static struct inode *jffs2_nfs_get_inode(struct super_block *sb, uint64_t ino,
                                         uint32_t generation)
{
        /* We don't care about i_generation. We'll destroy the flash
           before we start re-using inode numbers anyway. And even
           if that wasn't true, we'd have other problems...*/
        return jffs2_iget(sb, ino);
}

static struct dentry *jffs2_fh_to_dentry(struct super_block *sb, struct fid *fid,
                                         int fh_len, int fh_type)
{
        return generic_fh_to_dentry(sb, fid, fh_len, fh_type,
                                    jffs2_nfs_get_inode);
}

static struct dentry *jffs2_fh_to_parent(struct super_block *sb, struct fid *fid,
                                         int fh_len, int fh_type)
{
        return generic_fh_to_parent(sb, fid, fh_len, fh_type,
                                    jffs2_nfs_get_inode);
}

static struct dentry *jffs2_get_parent(struct dentry *child)
{
        struct jffs2_inode_info *f;
        uint32_t pino;

        BUG_ON(!d_is_dir(child));

        f = JFFS2_INODE_INFO(d_inode(child));

        pino = f->inocache->pino_nlink;

        JFFS2_DEBUG("Parent of directory ino #%u is #%u\n",
                    f->inocache->ino, pino);

        return d_obtain_alias(jffs2_iget(child->d_sb, pino));
}

static const struct export_operations jffs2_export_ops = {
        .encode_fh = generic_encode_ino32_fh,
        .get_parent = jffs2_get_parent,
        .fh_to_dentry = jffs2_fh_to_dentry,
        .fh_to_parent = jffs2_fh_to_parent,
};

/*
 * JFFS2 mount options.
 *
 * Opt_source: The source device
 * Opt_override_compr: override default compressor
 * Opt_rp_size: size of reserved pool in KiB
 */
enum {
        Opt_override_compr,
        Opt_rp_size,
};

static const struct constant_table jffs2_param_compr[] = {
        {"none",        JFFS2_COMPR_MODE_NONE },
#ifdef CONFIG_JFFS2_LZO
        {"lzo",                JFFS2_COMPR_MODE_FORCELZO },
#endif
#ifdef CONFIG_JFFS2_ZLIB
        {"zlib",        JFFS2_COMPR_MODE_FORCEZLIB },
#endif
        {}
};

static const struct fs_parameter_spec jffs2_fs_parameters[] = {
        fsparam_enum        ("compr",        Opt_override_compr, jffs2_param_compr),
        fsparam_u32        ("rp_size",        Opt_rp_size),
        {}
};

static int jffs2_parse_param(struct fs_context *fc, struct fs_parameter *param)
{
        struct fs_parse_result result;
        struct jffs2_sb_info *c = fc->s_fs_info;
        int opt;

        opt = fs_parse(fc, jffs2_fs_parameters, param, &result);
        if (opt < 0)
                return opt;

        switch (opt) {
        case Opt_override_compr:
                c->mount_opts.compr = result.uint_32;
                c->mount_opts.override_compr = true;
                break;
        case Opt_rp_size:
                if (result.uint_32 > UINT_MAX / 1024)
                        return invalf(fc, "jffs2: rp_size unrepresentable");
                c->mount_opts.rp_size = result.uint_32 * 1024;
                c->mount_opts.set_rp_size = true;
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

static inline void jffs2_update_mount_opts(struct fs_context *fc)
{
        struct jffs2_sb_info *new_c = fc->s_fs_info;
        struct jffs2_sb_info *c = JFFS2_SB_INFO(fc->root->d_sb);

        mutex_lock(&c->alloc_sem);
        if (new_c->mount_opts.override_compr) {
                c->mount_opts.override_compr = new_c->mount_opts.override_compr;
                c->mount_opts.compr = new_c->mount_opts.compr;
        }
        if (new_c->mount_opts.set_rp_size) {
                c->mount_opts.set_rp_size = new_c->mount_opts.set_rp_size;
                c->mount_opts.rp_size = new_c->mount_opts.rp_size;
        }
        mutex_unlock(&c->alloc_sem);
}

static int jffs2_reconfigure(struct fs_context *fc)
{
        struct super_block *sb = fc->root->d_sb;

        sync_filesystem(sb);
        jffs2_update_mount_opts(fc);

        return jffs2_do_remount_fs(sb, fc);
}

static const struct super_operations jffs2_super_operations =
{
        .alloc_inode =        jffs2_alloc_inode,
        .free_inode =        jffs2_free_inode,
        .put_super =        jffs2_put_super,
        .statfs =        jffs2_statfs,
        .evict_inode =        jffs2_evict_inode,
        .dirty_inode =        jffs2_dirty_inode,
        .show_options =        jffs2_show_options,
        .sync_fs =        jffs2_sync_fs,
};

/*
 * fill in the superblock
 */
static int jffs2_fill_super(struct super_block *sb, struct fs_context *fc)
{
        struct jffs2_sb_info *c = sb->s_fs_info;

        jffs2_dbg(1, "jffs2_get_sb_mtd():"
                  " New superblock for device %d (\"%s\")\n",
                  sb->s_mtd->index, sb->s_mtd->name);

        c->mtd = sb->s_mtd;
        c->os_priv = sb;

        if (c->mount_opts.rp_size > c->mtd->size)
                return invalf(fc, "jffs2: Too large reserve pool specified, max is %llu KB",
                              c->mtd->size / 1024);

        /* Initialize JFFS2 superblock locks, the further initialization will
         * be done later */
        mutex_init(&c->alloc_sem);
        mutex_init(&c->erase_free_sem);
        init_waitqueue_head(&c->erase_wait);
        init_waitqueue_head(&c->inocache_wq);
        spin_lock_init(&c->erase_completion_lock);
        spin_lock_init(&c->inocache_lock);

        sb->s_op = &jffs2_super_operations;
        sb->s_export_op = &jffs2_export_ops;
        sb->s_flags = sb->s_flags | SB_NOATIME;
        sb->s_xattr = jffs2_xattr_handlers;
#ifdef CONFIG_JFFS2_FS_POSIX_ACL
        sb->s_flags |= SB_POSIXACL;
#endif
        return jffs2_do_fill_super(sb, fc);
}

static int jffs2_get_tree(struct fs_context *fc)
{
        return get_tree_mtd(fc, jffs2_fill_super);
}

static void jffs2_free_fc(struct fs_context *fc)
{
        kfree(fc->s_fs_info);
}

static const struct fs_context_operations jffs2_context_ops = {
        .free                = jffs2_free_fc,
        .parse_param        = jffs2_parse_param,
        .get_tree        = jffs2_get_tree,
        .reconfigure        = jffs2_reconfigure,
};

static int jffs2_init_fs_context(struct fs_context *fc)
{
        struct jffs2_sb_info *ctx;

        ctx = kzalloc(sizeof(struct jffs2_sb_info), GFP_KERNEL);
        if (!ctx)
                return -ENOMEM;

        fc->s_fs_info = ctx;
        fc->ops = &jffs2_context_ops;
        return 0;
}

static void jffs2_put_super (struct super_block *sb)
{
        struct jffs2_sb_info *c = JFFS2_SB_INFO(sb);

        jffs2_dbg(2, "%s()\n", __func__);

        mutex_lock(&c->alloc_sem);
        jffs2_flush_wbuf_pad(c);
        mutex_unlock(&c->alloc_sem);

        jffs2_sum_exit(c);

        jffs2_free_ino_caches(c);
        jffs2_free_raw_node_refs(c);
        kvfree(c->blocks);
        jffs2_flash_cleanup(c);
        kfree(c->inocache_list);
        jffs2_clear_xattr_subsystem(c);
        mtd_sync(c->mtd);
        jffs2_dbg(1, "%s(): returning\n", __func__);
}

static void jffs2_kill_sb(struct super_block *sb)
{
        struct jffs2_sb_info *c = JFFS2_SB_INFO(sb);
        if (c && !sb_rdonly(sb))
                jffs2_stop_garbage_collect_thread(c);
        kill_mtd_super(sb);
        kfree(c);
}

static struct file_system_type jffs2_fs_type = {
        .owner =        THIS_MODULE,
        .name =                "jffs2",
        .init_fs_context = jffs2_init_fs_context,
        .parameters =        jffs2_fs_parameters,
        .kill_sb =        jffs2_kill_sb,
};
MODULE_ALIAS_FS("jffs2");

static int __init init_jffs2_fs(void)
{
        int ret;

        /* Paranoia checks for on-medium structures. If we ask GCC
           to pack them with __attribute__((packed)) then it _also_
           assumes that they're not aligned -- so it emits crappy
           code on some architectures. Ideally we want an attribute
           which means just 'no padding', without the alignment
           thing. But GCC doesn't have that -- we have to just
           hope the structs are the right sizes, instead. */
        BUILD_BUG_ON(sizeof(struct jffs2_unknown_node) != 12);
        BUILD_BUG_ON(sizeof(struct jffs2_raw_dirent) != 40);
        BUILD_BUG_ON(sizeof(struct jffs2_raw_inode) != 68);
        BUILD_BUG_ON(sizeof(struct jffs2_raw_summary) != 32);

        pr_info("version 2.2."
#ifdef CONFIG_JFFS2_FS_WRITEBUFFER
               " (NAND)"
#endif
#ifdef CONFIG_JFFS2_SUMMARY
               " (SUMMARY) "
#endif
               " © 2001-2006 Red Hat, Inc.\n");

        jffs2_inode_cachep = kmem_cache_create("jffs2_i",
                                             sizeof(struct jffs2_inode_info),
                                             0, (SLAB_RECLAIM_ACCOUNT|
                                                SLAB_ACCOUNT),
                                             jffs2_i_init_once);
        if (!jffs2_inode_cachep) {
                pr_err("error: Failed to initialise inode cache\n");
                return -ENOMEM;
        }
        ret = jffs2_compressors_init();
        if (ret) {
                pr_err("error: Failed to initialise compressors\n");
                goto out;
        }
        ret = jffs2_create_slab_caches();
        if (ret) {
                pr_err("error: Failed to initialise slab caches\n");
                goto out_compressors;
        }
        ret = register_filesystem(&jffs2_fs_type);
        if (ret) {
                pr_err("error: Failed to register filesystem\n");
                goto out_slab;
        }
        return 0;

 out_slab:
        jffs2_destroy_slab_caches();
 out_compressors:
        jffs2_compressors_exit();
 out:
        kmem_cache_destroy(jffs2_inode_cachep);
        return ret;
}

static void __exit exit_jffs2_fs(void)
{
        unregister_filesystem(&jffs2_fs_type);
        jffs2_destroy_slab_caches();
        jffs2_compressors_exit();

        /*
         * Make sure all delayed rcu free inodes are flushed before we
         * destroy cache.
         */
        rcu_barrier();
        kmem_cache_destroy(jffs2_inode_cachep);
}

module_init(init_jffs2_fs);
module_exit(exit_jffs2_fs);

MODULE_DESCRIPTION("The Journalling Flash File System, v2");
MODULE_AUTHOR("Red Hat, Inc.");
MODULE_LICENSE("GPL"); // Actually dual-licensed, but it doesn't matter for
                       // the sake of this tag. It's Free Software.



























   36 




   36 





   36 




































































   10 



























   10 






































































































































































    8 
    9 

    9 


    9 






    9 


   10 












































    9 












































    8 







    9 



   10 
   10 


   10 

























































































































































































































































































































































































































































































































    1 

    1 








    1 



    1 


























































































































































































































































































































































































































































































































































































































































































































































    1 

    1 
    1 

    1 



    1 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
// SPDX-License-Identifier: GPL-2.0
/*
 * This file contains helper code to handle channel
 * settings and keeping track of what is possible at
 * any point in time.
 *
 * Copyright 2009        Johannes Berg <johannes@sipsolutions.net>
 * Copyright 2013-2014  Intel Mobile Communications GmbH
 * Copyright 2018-2024        Intel Corporation
 */

#include <linux/export.h>
#include <linux/bitfield.h>
#include <net/cfg80211.h>
#include "core.h"
#include "rdev-ops.h"

static bool cfg80211_valid_60g_freq(u32 freq)
{
        return freq >= 58320 && freq <= 70200;
}

void cfg80211_chandef_create(struct cfg80211_chan_def *chandef,
                             struct ieee80211_channel *chan,
                             enum nl80211_channel_type chan_type)
{
        if (WARN_ON(!chan))
                return;

        *chandef = (struct cfg80211_chan_def) {
                .chan = chan,
                .freq1_offset = chan->freq_offset,
        };

        switch (chan_type) {
        case NL80211_CHAN_NO_HT:
                chandef->width = NL80211_CHAN_WIDTH_20_NOHT;
                chandef->center_freq1 = chan->center_freq;
                break;
        case NL80211_CHAN_HT20:
                chandef->width = NL80211_CHAN_WIDTH_20;
                chandef->center_freq1 = chan->center_freq;
                break;
        case NL80211_CHAN_HT40PLUS:
                chandef->width = NL80211_CHAN_WIDTH_40;
                chandef->center_freq1 = chan->center_freq + 10;
                break;
        case NL80211_CHAN_HT40MINUS:
                chandef->width = NL80211_CHAN_WIDTH_40;
                chandef->center_freq1 = chan->center_freq - 10;
                break;
        default:
                WARN_ON(1);
        }
}
EXPORT_SYMBOL(cfg80211_chandef_create);

struct cfg80211_per_bw_puncturing_values {
        u8 len;
        const u16 *valid_values;
};

static const u16 puncturing_values_80mhz[] = {
        0x8, 0x4, 0x2, 0x1
};

static const u16 puncturing_values_160mhz[] = {
         0x80, 0x40, 0x20, 0x10, 0x8, 0x4, 0x2, 0x1, 0xc0, 0x30, 0xc, 0x3
};

static const u16 puncturing_values_320mhz[] = {
        0xc000, 0x3000, 0xc00, 0x300, 0xc0, 0x30, 0xc, 0x3, 0xf000, 0xf00,
        0xf0, 0xf, 0xfc00, 0xf300, 0xf0c0, 0xf030, 0xf00c, 0xf003, 0xc00f,
        0x300f, 0xc0f, 0x30f, 0xcf, 0x3f
};

#define CFG80211_PER_BW_VALID_PUNCTURING_VALUES(_bw) \
        { \
                .len = ARRAY_SIZE(puncturing_values_ ## _bw ## mhz), \
                .valid_values = puncturing_values_ ## _bw ## mhz \
        }

static const struct cfg80211_per_bw_puncturing_values per_bw_puncturing[] = {
        CFG80211_PER_BW_VALID_PUNCTURING_VALUES(80),
        CFG80211_PER_BW_VALID_PUNCTURING_VALUES(160),
        CFG80211_PER_BW_VALID_PUNCTURING_VALUES(320)
};

static bool valid_puncturing_bitmap(const struct cfg80211_chan_def *chandef)
{
        u32 idx, i, start_freq, primary_center = chandef->chan->center_freq;

        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_80:
                idx = 0;
                start_freq = chandef->center_freq1 - 40;
                break;
        case NL80211_CHAN_WIDTH_160:
                idx = 1;
                start_freq = chandef->center_freq1 - 80;
                break;
        case NL80211_CHAN_WIDTH_320:
                idx = 2;
                start_freq = chandef->center_freq1 - 160;
                break;
        default:
                return chandef->punctured == 0;
        }

        if (!chandef->punctured)
                return true;

        /* check if primary channel is punctured */
        if (chandef->punctured & (u16)BIT((primary_center - start_freq) / 20))
                return false;

        for (i = 0; i < per_bw_puncturing[idx].len; i++) {
                if (per_bw_puncturing[idx].valid_values[i] == chandef->punctured)
                        return true;
        }

        return false;
}

static bool cfg80211_edmg_chandef_valid(const struct cfg80211_chan_def *chandef)
{
        int max_contiguous = 0;
        int num_of_enabled = 0;
        int contiguous = 0;
        int i;

        if (!chandef->edmg.channels || !chandef->edmg.bw_config)
                return false;

        if (!cfg80211_valid_60g_freq(chandef->chan->center_freq))
                return false;

        for (i = 0; i < 6; i++) {
                if (chandef->edmg.channels & BIT(i)) {
                        contiguous++;
                        num_of_enabled++;
                } else {
                        contiguous = 0;
                }

                max_contiguous = max(contiguous, max_contiguous);
        }
        /* basic verification of edmg configuration according to
         * IEEE P802.11ay/D4.0 section 9.4.2.251
         */
        /* check bw_config against contiguous edmg channels */
        switch (chandef->edmg.bw_config) {
        case IEEE80211_EDMG_BW_CONFIG_4:
        case IEEE80211_EDMG_BW_CONFIG_8:
        case IEEE80211_EDMG_BW_CONFIG_12:
                if (max_contiguous < 1)
                        return false;
                break;
        case IEEE80211_EDMG_BW_CONFIG_5:
        case IEEE80211_EDMG_BW_CONFIG_9:
        case IEEE80211_EDMG_BW_CONFIG_13:
                if (max_contiguous < 2)
                        return false;
                break;
        case IEEE80211_EDMG_BW_CONFIG_6:
        case IEEE80211_EDMG_BW_CONFIG_10:
        case IEEE80211_EDMG_BW_CONFIG_14:
                if (max_contiguous < 3)
                        return false;
                break;
        case IEEE80211_EDMG_BW_CONFIG_7:
        case IEEE80211_EDMG_BW_CONFIG_11:
        case IEEE80211_EDMG_BW_CONFIG_15:
                if (max_contiguous < 4)
                        return false;
                break;

        default:
                return false;
        }

        /* check bw_config against aggregated (non contiguous) edmg channels */
        switch (chandef->edmg.bw_config) {
        case IEEE80211_EDMG_BW_CONFIG_4:
        case IEEE80211_EDMG_BW_CONFIG_5:
        case IEEE80211_EDMG_BW_CONFIG_6:
        case IEEE80211_EDMG_BW_CONFIG_7:
                break;
        case IEEE80211_EDMG_BW_CONFIG_8:
        case IEEE80211_EDMG_BW_CONFIG_9:
        case IEEE80211_EDMG_BW_CONFIG_10:
        case IEEE80211_EDMG_BW_CONFIG_11:
                if (num_of_enabled < 2)
                        return false;
                break;
        case IEEE80211_EDMG_BW_CONFIG_12:
        case IEEE80211_EDMG_BW_CONFIG_13:
        case IEEE80211_EDMG_BW_CONFIG_14:
        case IEEE80211_EDMG_BW_CONFIG_15:
                if (num_of_enabled < 4 || max_contiguous < 2)
                        return false;
                break;
        default:
                return false;
        }

        return true;
}

int nl80211_chan_width_to_mhz(enum nl80211_chan_width chan_width)
{
        int mhz;

        switch (chan_width) {
        case NL80211_CHAN_WIDTH_1:
                mhz = 1;
                break;
        case NL80211_CHAN_WIDTH_2:
                mhz = 2;
                break;
        case NL80211_CHAN_WIDTH_4:
                mhz = 4;
                break;
        case NL80211_CHAN_WIDTH_8:
                mhz = 8;
                break;
        case NL80211_CHAN_WIDTH_16:
                mhz = 16;
                break;
        case NL80211_CHAN_WIDTH_5:
                mhz = 5;
                break;
        case NL80211_CHAN_WIDTH_10:
                mhz = 10;
                break;
        case NL80211_CHAN_WIDTH_20:
        case NL80211_CHAN_WIDTH_20_NOHT:
                mhz = 20;
                break;
        case NL80211_CHAN_WIDTH_40:
                mhz = 40;
                break;
        case NL80211_CHAN_WIDTH_80P80:
        case NL80211_CHAN_WIDTH_80:
                mhz = 80;
                break;
        case NL80211_CHAN_WIDTH_160:
                mhz = 160;
                break;
        case NL80211_CHAN_WIDTH_320:
                mhz = 320;
                break;
        default:
                WARN_ON_ONCE(1);
                return -1;
        }
        return mhz;
}
EXPORT_SYMBOL(nl80211_chan_width_to_mhz);

static int cfg80211_chandef_get_width(const struct cfg80211_chan_def *c)
{
        return nl80211_chan_width_to_mhz(c->width);
}

static bool cfg80211_valid_center_freq(u32 center,
                                       enum nl80211_chan_width width)
{
        int bw;
        int step;

        /* We only do strict verification on 6 GHz */
        if (center < 5955 || center > 7115)
                return true;

        bw = nl80211_chan_width_to_mhz(width);
        if (bw < 0)
                return false;

        /* Validate that the channels bw is entirely within the 6 GHz band */
        if (center - bw / 2 < 5945 || center + bw / 2 > 7125)
                return false;

        /* With 320 MHz the permitted channels overlap */
        if (bw == 320)
                step = 160;
        else
                step = bw;

        /*
         * Valid channels are packed from lowest frequency towards higher ones.
         * So test that the lower frequency aligns with one of these steps.
         */
        return (center - bw / 2 - 5945) % step == 0;
}

bool cfg80211_chandef_valid(const struct cfg80211_chan_def *chandef)
{
        u32 control_freq, oper_freq;
        int oper_width, control_width;

        if (!chandef->chan)
                return false;

        if (chandef->freq1_offset >= 1000)
                return false;

        control_freq = chandef->chan->center_freq;

        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_5:
        case NL80211_CHAN_WIDTH_10:
        case NL80211_CHAN_WIDTH_20:
        case NL80211_CHAN_WIDTH_20_NOHT:
                if (ieee80211_chandef_to_khz(chandef) !=
                    ieee80211_channel_to_khz(chandef->chan))
                        return false;
                if (chandef->center_freq2)
                        return false;
                break;
        case NL80211_CHAN_WIDTH_1:
        case NL80211_CHAN_WIDTH_2:
        case NL80211_CHAN_WIDTH_4:
        case NL80211_CHAN_WIDTH_8:
        case NL80211_CHAN_WIDTH_16:
                if (chandef->chan->band != NL80211_BAND_S1GHZ)
                        return false;

                control_freq = ieee80211_channel_to_khz(chandef->chan);
                oper_freq = ieee80211_chandef_to_khz(chandef);
                control_width = nl80211_chan_width_to_mhz(
                                        ieee80211_s1g_channel_width(
                                                                chandef->chan));
                oper_width = cfg80211_chandef_get_width(chandef);

                if (oper_width < 0 || control_width < 0)
                        return false;
                if (chandef->center_freq2)
                        return false;

                if (control_freq + MHZ_TO_KHZ(control_width) / 2 >
                    oper_freq + MHZ_TO_KHZ(oper_width) / 2)
                        return false;

                if (control_freq - MHZ_TO_KHZ(control_width) / 2 <
                    oper_freq - MHZ_TO_KHZ(oper_width) / 2)
                        return false;
                break;
        case NL80211_CHAN_WIDTH_80P80:
                if (!chandef->center_freq2)
                        return false;
                /* adjacent is not allowed -- that's a 160 MHz channel */
                if (chandef->center_freq1 - chandef->center_freq2 == 80 ||
                    chandef->center_freq2 - chandef->center_freq1 == 80)
                        return false;
                break;
        default:
                if (chandef->center_freq2)
                        return false;
                break;
        }

        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_5:
        case NL80211_CHAN_WIDTH_10:
        case NL80211_CHAN_WIDTH_20:
        case NL80211_CHAN_WIDTH_20_NOHT:
        case NL80211_CHAN_WIDTH_1:
        case NL80211_CHAN_WIDTH_2:
        case NL80211_CHAN_WIDTH_4:
        case NL80211_CHAN_WIDTH_8:
        case NL80211_CHAN_WIDTH_16:
                /* all checked above */
                break;
        case NL80211_CHAN_WIDTH_320:
                if (chandef->center_freq1 == control_freq + 150 ||
                    chandef->center_freq1 == control_freq + 130 ||
                    chandef->center_freq1 == control_freq + 110 ||
                    chandef->center_freq1 == control_freq + 90 ||
                    chandef->center_freq1 == control_freq - 90 ||
                    chandef->center_freq1 == control_freq - 110 ||
                    chandef->center_freq1 == control_freq - 130 ||
                    chandef->center_freq1 == control_freq - 150)
                        break;
                fallthrough;
        case NL80211_CHAN_WIDTH_160:
                if (chandef->center_freq1 == control_freq + 70 ||
                    chandef->center_freq1 == control_freq + 50 ||
                    chandef->center_freq1 == control_freq - 50 ||
                    chandef->center_freq1 == control_freq - 70)
                        break;
                fallthrough;
        case NL80211_CHAN_WIDTH_80P80:
        case NL80211_CHAN_WIDTH_80:
                if (chandef->center_freq1 == control_freq + 30 ||
                    chandef->center_freq1 == control_freq - 30)
                        break;
                fallthrough;
        case NL80211_CHAN_WIDTH_40:
                if (chandef->center_freq1 == control_freq + 10 ||
                    chandef->center_freq1 == control_freq - 10)
                        break;
                fallthrough;
        default:
                return false;
        }

        if (!cfg80211_valid_center_freq(chandef->center_freq1, chandef->width))
                return false;

        if (chandef->width == NL80211_CHAN_WIDTH_80P80 &&
            !cfg80211_valid_center_freq(chandef->center_freq2, chandef->width))
                return false;

        /* channel 14 is only for IEEE 802.11b */
        if (chandef->center_freq1 == 2484 &&
            chandef->width != NL80211_CHAN_WIDTH_20_NOHT)
                return false;

        if (cfg80211_chandef_is_edmg(chandef) &&
            !cfg80211_edmg_chandef_valid(chandef))
                return false;

        return valid_puncturing_bitmap(chandef);
}
EXPORT_SYMBOL(cfg80211_chandef_valid);

int cfg80211_chandef_primary(const struct cfg80211_chan_def *c,
                             enum nl80211_chan_width primary_chan_width,
                             u16 *punctured)
{
        int pri_width = nl80211_chan_width_to_mhz(primary_chan_width);
        int width = cfg80211_chandef_get_width(c);
        u32 control = c->chan->center_freq;
        u32 center = c->center_freq1;
        u16 _punct = 0;

        if (WARN_ON_ONCE(pri_width < 0 || width < 0))
                return -1;

        /* not intended to be called this way, can't determine */
        if (WARN_ON_ONCE(pri_width > width))
                return -1;

        if (!punctured)
                punctured = &_punct;

        *punctured = c->punctured;

        while (width > pri_width) {
                unsigned int bits_to_drop = width / 20 / 2;

                if (control > center) {
                        center += width / 4;
                        *punctured >>= bits_to_drop;
                } else {
                        center -= width / 4;
                        *punctured &= (1 << bits_to_drop) - 1;
                }
                width /= 2;
        }

        return center;
}
EXPORT_SYMBOL(cfg80211_chandef_primary);

static const struct cfg80211_chan_def *
check_chandef_primary_compat(const struct cfg80211_chan_def *c1,
                             const struct cfg80211_chan_def *c2,
                             enum nl80211_chan_width primary_chan_width)
{
        u16 punct_c1 = 0, punct_c2 = 0;

        /* check primary is compatible -> error if not */
        if (cfg80211_chandef_primary(c1, primary_chan_width, &punct_c1) !=
            cfg80211_chandef_primary(c2, primary_chan_width, &punct_c2))
                return ERR_PTR(-EINVAL);

        if (punct_c1 != punct_c2)
                return ERR_PTR(-EINVAL);

        /* assumes c1 is smaller width, if that was just checked -> done */
        if (c1->width == primary_chan_width)
                return c2;

        /* otherwise continue checking the next width */
        return NULL;
}

static const struct cfg80211_chan_def *
_cfg80211_chandef_compatible(const struct cfg80211_chan_def *c1,
                             const struct cfg80211_chan_def *c2)
{
        const struct cfg80211_chan_def *ret;

        /* If they are identical, return */
        if (cfg80211_chandef_identical(c1, c2))
                return c2;

        /* otherwise, must have same control channel */
        if (c1->chan != c2->chan)
                return NULL;

        /*
         * If they have the same width, but aren't identical,
         * then they can't be compatible.
         */
        if (c1->width == c2->width)
                return NULL;

        /*
         * can't be compatible if one of them is 5/10 MHz or S1G
         * but they don't have the same width.
         */
#define NARROW_OR_S1G(width)        ((width) == NL80211_CHAN_WIDTH_5 || \
                                 (width) == NL80211_CHAN_WIDTH_10 || \
                                 (width) == NL80211_CHAN_WIDTH_1 || \
                                 (width) == NL80211_CHAN_WIDTH_2 || \
                                 (width) == NL80211_CHAN_WIDTH_4 || \
                                 (width) == NL80211_CHAN_WIDTH_8 || \
                                 (width) == NL80211_CHAN_WIDTH_16)

        if (NARROW_OR_S1G(c1->width) || NARROW_OR_S1G(c2->width))
                return NULL;

        /*
         * Make sure that c1 is always the narrower one, so that later
         * we either return NULL or c2 and don't have to check both
         * directions.
         */
        if (c1->width > c2->width)
                swap(c1, c2);

        /*
         * No further checks needed if the "narrower" one is only 20 MHz.
         * Here "narrower" includes being a 20 MHz non-HT channel vs. a
         * 20 MHz HT (or later) one.
         */
        if (c1->width <= NL80211_CHAN_WIDTH_20)
                return c2;

        ret = check_chandef_primary_compat(c1, c2, NL80211_CHAN_WIDTH_40);
        if (ret)
                return ret;

        ret = check_chandef_primary_compat(c1, c2, NL80211_CHAN_WIDTH_80);
        if (ret)
                return ret;

        /*
         * If c1 is 80+80, then c2 is 160 or higher, but that cannot
         * match. If c2 was also 80+80 it was already either accepted
         * or rejected above (identical or not, respectively.)
         */
        if (c1->width == NL80211_CHAN_WIDTH_80P80)
                return NULL;

        ret = check_chandef_primary_compat(c1, c2, NL80211_CHAN_WIDTH_160);
        if (ret)
                return ret;

        /*
         * Getting here would mean they're both wider than 160, have the
         * same primary 160, but are not identical - this cannot happen
         * since they must be 320 (no wider chandefs exist, at least yet.)
         */
        WARN_ON_ONCE(1);

        return NULL;
}

const struct cfg80211_chan_def *
cfg80211_chandef_compatible(const struct cfg80211_chan_def *c1,
                            const struct cfg80211_chan_def *c2)
{
        const struct cfg80211_chan_def *ret;

        ret = _cfg80211_chandef_compatible(c1, c2);
        if (IS_ERR(ret))
                return NULL;
        return ret;
}
EXPORT_SYMBOL(cfg80211_chandef_compatible);

static void cfg80211_set_chans_dfs_state(struct wiphy *wiphy, u32 center_freq,
                                         u32 bandwidth,
                                         enum nl80211_dfs_state dfs_state)
{
        struct ieee80211_channel *c;
        u32 freq;

        for (freq = center_freq - bandwidth/2 + 10;
             freq <= center_freq + bandwidth/2 - 10;
             freq += 20) {
                c = ieee80211_get_channel(wiphy, freq);
                if (!c || !(c->flags & IEEE80211_CHAN_RADAR))
                        continue;

                c->dfs_state = dfs_state;
                c->dfs_state_entered = jiffies;
        }
}

void cfg80211_set_dfs_state(struct wiphy *wiphy,
                            const struct cfg80211_chan_def *chandef,
                            enum nl80211_dfs_state dfs_state)
{
        int width;

        if (WARN_ON(!cfg80211_chandef_valid(chandef)))
                return;

        width = cfg80211_chandef_get_width(chandef);
        if (width < 0)
                return;

        cfg80211_set_chans_dfs_state(wiphy, chandef->center_freq1,
                                     width, dfs_state);

        if (!chandef->center_freq2)
                return;
        cfg80211_set_chans_dfs_state(wiphy, chandef->center_freq2,
                                     width, dfs_state);
}

static u32 cfg80211_get_start_freq(u32 center_freq,
                                   u32 bandwidth)
{
        u32 start_freq;

        bandwidth = MHZ_TO_KHZ(bandwidth);
        if (bandwidth <= MHZ_TO_KHZ(20))
                start_freq = center_freq;
        else
                start_freq = center_freq - bandwidth / 2 + MHZ_TO_KHZ(10);

        return start_freq;
}

static u32 cfg80211_get_end_freq(u32 center_freq,
                                 u32 bandwidth)
{
        u32 end_freq;

        bandwidth = MHZ_TO_KHZ(bandwidth);
        if (bandwidth <= MHZ_TO_KHZ(20))
                end_freq = center_freq;
        else
                end_freq = center_freq + bandwidth / 2 - MHZ_TO_KHZ(10);

        return end_freq;
}

static bool
cfg80211_dfs_permissive_check_wdev(struct cfg80211_registered_device *rdev,
                                   enum nl80211_iftype iftype,
                                   struct wireless_dev *wdev,
                                   struct ieee80211_channel *chan)
{
        unsigned int link_id;

        for_each_valid_link(wdev, link_id) {
                struct ieee80211_channel *other_chan = NULL;
                struct cfg80211_chan_def chandef = {};
                int ret;

                /* In order to avoid daisy chaining only allow BSS STA */
                if (wdev->iftype != NL80211_IFTYPE_STATION ||
                    !wdev->links[link_id].client.current_bss)
                        continue;

                other_chan =
                        wdev->links[link_id].client.current_bss->pub.channel;

                if (!other_chan)
                        continue;

                if (chan == other_chan)
                        return true;

                /* continue if we can't get the channel */
                ret = rdev_get_channel(rdev, wdev, link_id, &chandef);
                if (ret)
                        continue;

                if (cfg80211_is_sub_chan(&chandef, chan, false))
                        return true;
        }

        return false;
}

/*
 * Check if P2P GO is allowed to operate on a DFS channel
 */
static bool cfg80211_dfs_permissive_chan(struct wiphy *wiphy,
                                         enum nl80211_iftype iftype,
                                         struct ieee80211_channel *chan)
{
        struct wireless_dev *wdev;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);

        lockdep_assert_held(&rdev->wiphy.mtx);

        if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_DFS_CONCURRENT) ||
            !(chan->flags & IEEE80211_CHAN_DFS_CONCURRENT))
                return false;

        /* only valid for P2P GO */
        if (iftype != NL80211_IFTYPE_P2P_GO)
                return false;

        /*
         * Allow only if there's a concurrent BSS
         */
        list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                bool ret = cfg80211_dfs_permissive_check_wdev(rdev, iftype,
                                                              wdev, chan);
                if (ret)
                        return ret;
        }

        return false;
}

static int cfg80211_get_chans_dfs_required(struct wiphy *wiphy,
                                            u32 center_freq,
                                            u32 bandwidth,
                                            enum nl80211_iftype iftype)
{
        struct ieee80211_channel *c;
        u32 freq, start_freq, end_freq;

        start_freq = cfg80211_get_start_freq(center_freq, bandwidth);
        end_freq = cfg80211_get_end_freq(center_freq, bandwidth);

        for (freq = start_freq; freq <= end_freq; freq += MHZ_TO_KHZ(20)) {
                c = ieee80211_get_channel_khz(wiphy, freq);
                if (!c)
                        return -EINVAL;

                if (c->flags & IEEE80211_CHAN_RADAR &&
                    !cfg80211_dfs_permissive_chan(wiphy, iftype, c))
                        return 1;
        }

        return 0;
}


int cfg80211_chandef_dfs_required(struct wiphy *wiphy,
                                  const struct cfg80211_chan_def *chandef,
                                  enum nl80211_iftype iftype)
{
        int width;
        int ret;

        if (WARN_ON(!cfg80211_chandef_valid(chandef)))
                return -EINVAL;

        switch (iftype) {
        case NL80211_IFTYPE_ADHOC:
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_P2P_GO:
        case NL80211_IFTYPE_MESH_POINT:
                width = cfg80211_chandef_get_width(chandef);
                if (width < 0)
                        return -EINVAL;

                ret = cfg80211_get_chans_dfs_required(wiphy,
                                        ieee80211_chandef_to_khz(chandef),
                                        width, iftype);
                if (ret < 0)
                        return ret;
                else if (ret > 0)
                        return BIT(chandef->width);

                if (!chandef->center_freq2)
                        return 0;

                ret = cfg80211_get_chans_dfs_required(wiphy,
                                        MHZ_TO_KHZ(chandef->center_freq2),
                                        width, iftype);
                if (ret < 0)
                        return ret;
                else if (ret > 0)
                        return BIT(chandef->width);

                break;
        case NL80211_IFTYPE_STATION:
        case NL80211_IFTYPE_OCB:
        case NL80211_IFTYPE_P2P_CLIENT:
        case NL80211_IFTYPE_MONITOR:
        case NL80211_IFTYPE_AP_VLAN:
        case NL80211_IFTYPE_P2P_DEVICE:
        case NL80211_IFTYPE_NAN:
                break;
        case NL80211_IFTYPE_WDS:
        case NL80211_IFTYPE_UNSPECIFIED:
        case NUM_NL80211_IFTYPES:
                WARN_ON(1);
        }

        return 0;
}
EXPORT_SYMBOL(cfg80211_chandef_dfs_required);

static int cfg80211_get_chans_dfs_usable(struct wiphy *wiphy,
                                         u32 center_freq,
                                         u32 bandwidth)
{
        struct ieee80211_channel *c;
        u32 freq, start_freq, end_freq;
        int count = 0;

        start_freq = cfg80211_get_start_freq(center_freq, bandwidth);
        end_freq = cfg80211_get_end_freq(center_freq, bandwidth);

        /*
         * Check entire range of channels for the bandwidth.
         * Check all channels are DFS channels (DFS_USABLE or
         * DFS_AVAILABLE). Return number of usable channels
         * (require CAC). Allow DFS and non-DFS channel mix.
         */
        for (freq = start_freq; freq <= end_freq; freq += MHZ_TO_KHZ(20)) {
                c = ieee80211_get_channel_khz(wiphy, freq);
                if (!c)
                        return -EINVAL;

                if (c->flags & IEEE80211_CHAN_DISABLED)
                        return -EINVAL;

                if (c->flags & IEEE80211_CHAN_RADAR) {
                        if (c->dfs_state == NL80211_DFS_UNAVAILABLE)
                                return -EINVAL;

                        if (c->dfs_state == NL80211_DFS_USABLE)
                                count++;
                }
        }

        return count;
}

bool cfg80211_chandef_dfs_usable(struct wiphy *wiphy,
                                 const struct cfg80211_chan_def *chandef)
{
        int width;
        int r1, r2 = 0;

        if (WARN_ON(!cfg80211_chandef_valid(chandef)))
                return false;

        width = cfg80211_chandef_get_width(chandef);
        if (width < 0)
                return false;

        r1 = cfg80211_get_chans_dfs_usable(wiphy,
                                           MHZ_TO_KHZ(chandef->center_freq1),
                                           width);

        if (r1 < 0)
                return false;

        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_80P80:
                WARN_ON(!chandef->center_freq2);
                r2 = cfg80211_get_chans_dfs_usable(wiphy,
                                        MHZ_TO_KHZ(chandef->center_freq2),
                                        width);
                if (r2 < 0)
                        return false;
                break;
        default:
                WARN_ON(chandef->center_freq2);
                break;
        }

        return (r1 + r2 > 0);
}
EXPORT_SYMBOL(cfg80211_chandef_dfs_usable);

/*
 * Checks if center frequency of chan falls with in the bandwidth
 * range of chandef.
 */
bool cfg80211_is_sub_chan(struct cfg80211_chan_def *chandef,
                          struct ieee80211_channel *chan,
                          bool primary_only)
{
        int width;
        u32 freq;

        if (!chandef->chan)
                return false;

        if (chandef->chan->center_freq == chan->center_freq)
                return true;

        if (primary_only)
                return false;

        width = cfg80211_chandef_get_width(chandef);
        if (width <= 20)
                return false;

        for (freq = chandef->center_freq1 - width / 2 + 10;
             freq <= chandef->center_freq1 + width / 2 - 10; freq += 20) {
                if (chan->center_freq == freq)
                        return true;
        }

        if (!chandef->center_freq2)
                return false;

        for (freq = chandef->center_freq2 - width / 2 + 10;
             freq <= chandef->center_freq2 + width / 2 - 10; freq += 20) {
                if (chan->center_freq == freq)
                        return true;
        }

        return false;
}

bool cfg80211_beaconing_iface_active(struct wireless_dev *wdev)
{
        unsigned int link;

        lockdep_assert_wiphy(wdev->wiphy);

        switch (wdev->iftype) {
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_P2P_GO:
                for_each_valid_link(wdev, link) {
                        if (wdev->links[link].ap.beacon_interval)
                                return true;
                }
                break;
        case NL80211_IFTYPE_ADHOC:
                if (wdev->u.ibss.ssid_len)
                        return true;
                break;
        case NL80211_IFTYPE_MESH_POINT:
                if (wdev->u.mesh.id_len)
                        return true;
                break;
        case NL80211_IFTYPE_STATION:
        case NL80211_IFTYPE_OCB:
        case NL80211_IFTYPE_P2P_CLIENT:
        case NL80211_IFTYPE_MONITOR:
        case NL80211_IFTYPE_AP_VLAN:
        case NL80211_IFTYPE_P2P_DEVICE:
        /* Can NAN type be considered as beaconing interface? */
        case NL80211_IFTYPE_NAN:
                break;
        case NL80211_IFTYPE_UNSPECIFIED:
        case NL80211_IFTYPE_WDS:
        case NUM_NL80211_IFTYPES:
                WARN_ON(1);
        }

        return false;
}

bool cfg80211_wdev_on_sub_chan(struct wireless_dev *wdev,
                               struct ieee80211_channel *chan,
                               bool primary_only)
{
        unsigned int link;

        switch (wdev->iftype) {
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_P2P_GO:
                for_each_valid_link(wdev, link) {
                        if (cfg80211_is_sub_chan(&wdev->links[link].ap.chandef,
                                                 chan, primary_only))
                                return true;
                }
                break;
        case NL80211_IFTYPE_ADHOC:
                return cfg80211_is_sub_chan(&wdev->u.ibss.chandef, chan,
                                            primary_only);
        case NL80211_IFTYPE_MESH_POINT:
                return cfg80211_is_sub_chan(&wdev->u.mesh.chandef, chan,
                                            primary_only);
        default:
                break;
        }

        return false;
}

static bool cfg80211_is_wiphy_oper_chan(struct wiphy *wiphy,
                                        struct ieee80211_channel *chan)
{
        struct wireless_dev *wdev;

        lockdep_assert_wiphy(wiphy);

        list_for_each_entry(wdev, &wiphy->wdev_list, list) {
                if (!cfg80211_beaconing_iface_active(wdev))
                        continue;

                if (cfg80211_wdev_on_sub_chan(wdev, chan, false))
                        return true;
        }

        return false;
}

static bool
cfg80211_offchan_chain_is_active(struct cfg80211_registered_device *rdev,
                                 struct ieee80211_channel *channel)
{
        if (!rdev->background_radar_wdev)
                return false;

        if (!cfg80211_chandef_valid(&rdev->background_radar_chandef))
                return false;

        return cfg80211_is_sub_chan(&rdev->background_radar_chandef, channel,
                                    false);
}

bool cfg80211_any_wiphy_oper_chan(struct wiphy *wiphy,
                                  struct ieee80211_channel *chan)
{
        struct cfg80211_registered_device *rdev;

        ASSERT_RTNL();

        if (!(chan->flags & IEEE80211_CHAN_RADAR))
                return false;

        for_each_rdev(rdev) {
                bool found;

                if (!reg_dfs_domain_same(wiphy, &rdev->wiphy))
                        continue;

                wiphy_lock(&rdev->wiphy);
                found = cfg80211_is_wiphy_oper_chan(&rdev->wiphy, chan) ||
                        cfg80211_offchan_chain_is_active(rdev, chan);
                wiphy_unlock(&rdev->wiphy);

                if (found)
                        return true;
        }

        return false;
}

static bool cfg80211_get_chans_dfs_available(struct wiphy *wiphy,
                                             u32 center_freq,
                                             u32 bandwidth)
{
        struct ieee80211_channel *c;
        u32 freq, start_freq, end_freq;
        bool dfs_offload;

        dfs_offload = wiphy_ext_feature_isset(wiphy,
                                              NL80211_EXT_FEATURE_DFS_OFFLOAD);

        start_freq = cfg80211_get_start_freq(center_freq, bandwidth);
        end_freq = cfg80211_get_end_freq(center_freq, bandwidth);

        /*
         * Check entire range of channels for the bandwidth.
         * If any channel in between is disabled or has not
         * had gone through CAC return false
         */
        for (freq = start_freq; freq <= end_freq; freq += MHZ_TO_KHZ(20)) {
                c = ieee80211_get_channel_khz(wiphy, freq);
                if (!c)
                        return false;

                if (c->flags & IEEE80211_CHAN_DISABLED)
                        return false;

                if ((c->flags & IEEE80211_CHAN_RADAR) &&
                    (c->dfs_state != NL80211_DFS_AVAILABLE) &&
                    !(c->dfs_state == NL80211_DFS_USABLE && dfs_offload))
                        return false;
        }

        return true;
}

static bool cfg80211_chandef_dfs_available(struct wiphy *wiphy,
                                const struct cfg80211_chan_def *chandef)
{
        int width;
        int r;

        if (WARN_ON(!cfg80211_chandef_valid(chandef)))
                return false;

        width = cfg80211_chandef_get_width(chandef);
        if (width < 0)
                return false;

        r = cfg80211_get_chans_dfs_available(wiphy,
                                             MHZ_TO_KHZ(chandef->center_freq1),
                                             width);

        /* If any of channels unavailable for cf1 just return */
        if (!r)
                return r;

        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_80P80:
                WARN_ON(!chandef->center_freq2);
                r = cfg80211_get_chans_dfs_available(wiphy,
                                        MHZ_TO_KHZ(chandef->center_freq2),
                                        width);
                break;
        default:
                WARN_ON(chandef->center_freq2);
                break;
        }

        return r;
}

static unsigned int cfg80211_get_chans_dfs_cac_time(struct wiphy *wiphy,
                                                    u32 center_freq,
                                                    u32 bandwidth)
{
        struct ieee80211_channel *c;
        u32 start_freq, end_freq, freq;
        unsigned int dfs_cac_ms = 0;

        start_freq = cfg80211_get_start_freq(center_freq, bandwidth);
        end_freq = cfg80211_get_end_freq(center_freq, bandwidth);

        for (freq = start_freq; freq <= end_freq; freq += MHZ_TO_KHZ(20)) {
                c = ieee80211_get_channel_khz(wiphy, freq);
                if (!c)
                        return 0;

                if (c->flags & IEEE80211_CHAN_DISABLED)
                        return 0;

                if (!(c->flags & IEEE80211_CHAN_RADAR))
                        continue;

                if (c->dfs_cac_ms > dfs_cac_ms)
                        dfs_cac_ms = c->dfs_cac_ms;
        }

        return dfs_cac_ms;
}

unsigned int
cfg80211_chandef_dfs_cac_time(struct wiphy *wiphy,
                              const struct cfg80211_chan_def *chandef)
{
        int width;
        unsigned int t1 = 0, t2 = 0;

        if (WARN_ON(!cfg80211_chandef_valid(chandef)))
                return 0;

        width = cfg80211_chandef_get_width(chandef);
        if (width < 0)
                return 0;

        t1 = cfg80211_get_chans_dfs_cac_time(wiphy,
                                             MHZ_TO_KHZ(chandef->center_freq1),
                                             width);

        if (!chandef->center_freq2)
                return t1;

        t2 = cfg80211_get_chans_dfs_cac_time(wiphy,
                                             MHZ_TO_KHZ(chandef->center_freq2),
                                             width);

        return max(t1, t2);
}
EXPORT_SYMBOL(cfg80211_chandef_dfs_cac_time);

static bool cfg80211_secondary_chans_ok(struct wiphy *wiphy,
                                        u32 center_freq, u32 bandwidth,
                                        u32 prohibited_flags,
                                        u32 permitting_flags)
{
        struct ieee80211_channel *c;
        u32 freq, start_freq, end_freq;

        start_freq = cfg80211_get_start_freq(center_freq, bandwidth);
        end_freq = cfg80211_get_end_freq(center_freq, bandwidth);

        for (freq = start_freq; freq <= end_freq; freq += MHZ_TO_KHZ(20)) {
                c = ieee80211_get_channel_khz(wiphy, freq);
                if (!c)
                        return false;
                if (c->flags & permitting_flags)
                        continue;
                if (c->flags & prohibited_flags)
                        return false;
        }

        return true;
}

/* check if the operating channels are valid and supported */
static bool cfg80211_edmg_usable(struct wiphy *wiphy, u8 edmg_channels,
                                 enum ieee80211_edmg_bw_config edmg_bw_config,
                                 int primary_channel,
                                 struct ieee80211_edmg *edmg_cap)
{
        struct ieee80211_channel *chan;
        int i, freq;
        int channels_counter = 0;

        if (!edmg_channels && !edmg_bw_config)
                return true;

        if ((!edmg_channels && edmg_bw_config) ||
            (edmg_channels && !edmg_bw_config))
                return false;

        if (!(edmg_channels & BIT(primary_channel - 1)))
                return false;

        /* 60GHz channels 1..6 */
        for (i = 0; i < 6; i++) {
                if (!(edmg_channels & BIT(i)))
                        continue;

                if (!(edmg_cap->channels & BIT(i)))
                        return false;

                channels_counter++;

                freq = ieee80211_channel_to_frequency(i + 1,
                                                      NL80211_BAND_60GHZ);
                chan = ieee80211_get_channel(wiphy, freq);
                if (!chan || chan->flags & IEEE80211_CHAN_DISABLED)
                        return false;
        }

        /* IEEE802.11 allows max 4 channels */
        if (channels_counter > 4)
                return false;

        /* check bw_config is a subset of what driver supports
         * (see IEEE P802.11ay/D4.0 section 9.4.2.251, Table 13)
         */
        if ((edmg_bw_config % 4) > (edmg_cap->bw_config % 4))
                return false;

        if (edmg_bw_config > edmg_cap->bw_config)
                return false;

        return true;
}

bool _cfg80211_chandef_usable(struct wiphy *wiphy,
                              const struct cfg80211_chan_def *chandef,
                              u32 prohibited_flags,
                              u32 permitting_flags)
{
        struct ieee80211_sta_ht_cap *ht_cap;
        struct ieee80211_sta_vht_cap *vht_cap;
        struct ieee80211_edmg *edmg_cap;
        u32 width, control_freq, cap;
        bool ext_nss_cap, support_80_80 = false, support_320 = false;
        const struct ieee80211_sband_iftype_data *iftd;
        struct ieee80211_supported_band *sband;
        int i;

        if (WARN_ON(!cfg80211_chandef_valid(chandef)))
                return false;

        ht_cap = &wiphy->bands[chandef->chan->band]->ht_cap;
        vht_cap = &wiphy->bands[chandef->chan->band]->vht_cap;
        edmg_cap = &wiphy->bands[chandef->chan->band]->edmg_cap;
        ext_nss_cap = __le16_to_cpu(vht_cap->vht_mcs.tx_highest) &
                        IEEE80211_VHT_EXT_NSS_BW_CAPABLE;

        if (edmg_cap->channels &&
            !cfg80211_edmg_usable(wiphy,
                                  chandef->edmg.channels,
                                  chandef->edmg.bw_config,
                                  chandef->chan->hw_value,
                                  edmg_cap))
                return false;

        control_freq = chandef->chan->center_freq;

        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_1:
                width = 1;
                break;
        case NL80211_CHAN_WIDTH_2:
                width = 2;
                break;
        case NL80211_CHAN_WIDTH_4:
                width = 4;
                break;
        case NL80211_CHAN_WIDTH_8:
                width = 8;
                break;
        case NL80211_CHAN_WIDTH_16:
                width = 16;
                break;
        case NL80211_CHAN_WIDTH_5:
                width = 5;
                break;
        case NL80211_CHAN_WIDTH_10:
                prohibited_flags |= IEEE80211_CHAN_NO_10MHZ;
                width = 10;
                break;
        case NL80211_CHAN_WIDTH_20:
                if (!ht_cap->ht_supported &&
                    chandef->chan->band != NL80211_BAND_6GHZ)
                        return false;
                fallthrough;
        case NL80211_CHAN_WIDTH_20_NOHT:
                prohibited_flags |= IEEE80211_CHAN_NO_20MHZ;
                width = 20;
                break;
        case NL80211_CHAN_WIDTH_40:
                width = 40;
                if (chandef->chan->band == NL80211_BAND_6GHZ)
                        break;
                if (!ht_cap->ht_supported)
                        return false;
                if (!(ht_cap->cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40) ||
                    ht_cap->cap & IEEE80211_HT_CAP_40MHZ_INTOLERANT)
                        return false;
                if (chandef->center_freq1 < control_freq &&
                    chandef->chan->flags & IEEE80211_CHAN_NO_HT40MINUS)
                        return false;
                if (chandef->center_freq1 > control_freq &&
                    chandef->chan->flags & IEEE80211_CHAN_NO_HT40PLUS)
                        return false;
                break;
        case NL80211_CHAN_WIDTH_80P80:
                cap = vht_cap->cap;
                support_80_80 =
                        (cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ) ||
                        (cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ &&
                         cap & IEEE80211_VHT_CAP_EXT_NSS_BW_MASK) ||
                        (ext_nss_cap &&
                         u32_get_bits(cap, IEEE80211_VHT_CAP_EXT_NSS_BW_MASK) > 1);
                if (chandef->chan->band != NL80211_BAND_6GHZ && !support_80_80)
                        return false;
                fallthrough;
        case NL80211_CHAN_WIDTH_80:
                prohibited_flags |= IEEE80211_CHAN_NO_80MHZ;
                width = 80;
                if (chandef->chan->band == NL80211_BAND_6GHZ)
                        break;
                if (!vht_cap->vht_supported)
                        return false;
                break;
        case NL80211_CHAN_WIDTH_160:
                prohibited_flags |= IEEE80211_CHAN_NO_160MHZ;
                width = 160;
                if (chandef->chan->band == NL80211_BAND_6GHZ)
                        break;
                if (!vht_cap->vht_supported)
                        return false;
                cap = vht_cap->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;
                if (cap != IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ &&
                    cap != IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ &&
                    !(ext_nss_cap &&
                      (vht_cap->cap & IEEE80211_VHT_CAP_EXT_NSS_BW_MASK)))
                        return false;
                break;
        case NL80211_CHAN_WIDTH_320:
                prohibited_flags |= IEEE80211_CHAN_NO_320MHZ;
                width = 320;

                if (chandef->chan->band != NL80211_BAND_6GHZ)
                        return false;

                sband = wiphy->bands[NL80211_BAND_6GHZ];
                if (!sband)
                        return false;

                for_each_sband_iftype_data(sband, i, iftd) {
                        if (!iftd->eht_cap.has_eht)
                                continue;

                        if (iftd->eht_cap.eht_cap_elem.phy_cap_info[0] &
                            IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ) {
                                support_320 = true;
                                break;
                        }
                }

                if (!support_320)
                        return false;
                break;
        default:
                WARN_ON_ONCE(1);
                return false;
        }

        /*
         * TODO: What if there are only certain 80/160/80+80 MHz channels
         *         allowed by the driver, or only certain combinations?
         *         For 40 MHz the driver can set the NO_HT40 flags, but for
         *         80/160 MHz and in particular 80+80 MHz this isn't really
         *         feasible and we only have NO_80MHZ/NO_160MHZ so far but
         *         no way to cover 80+80 MHz or more complex restrictions.
         *         Note that such restrictions also need to be advertised to
         *         userspace, for example for P2P channel selection.
         */

        if (width > 20)
                prohibited_flags |= IEEE80211_CHAN_NO_OFDM;

        /* 5 and 10 MHz are only defined for the OFDM PHY */
        if (width < 20)
                prohibited_flags |= IEEE80211_CHAN_NO_OFDM;


        if (!cfg80211_secondary_chans_ok(wiphy,
                                         ieee80211_chandef_to_khz(chandef),
                                         width, prohibited_flags,
                                         permitting_flags))
                return false;

        if (!chandef->center_freq2)
                return true;
        return cfg80211_secondary_chans_ok(wiphy,
                                           MHZ_TO_KHZ(chandef->center_freq2),
                                           width, prohibited_flags,
                                           permitting_flags);
}

bool cfg80211_chandef_usable(struct wiphy *wiphy,
                             const struct cfg80211_chan_def *chandef,
                             u32 prohibited_flags)
{
        return _cfg80211_chandef_usable(wiphy, chandef, prohibited_flags, 0);
}
EXPORT_SYMBOL(cfg80211_chandef_usable);

static bool cfg80211_ir_permissive_check_wdev(enum nl80211_iftype iftype,
                                              struct wireless_dev *wdev,
                                              struct ieee80211_channel *chan)
{
        struct ieee80211_channel *other_chan = NULL;
        unsigned int link_id;
        int r1, r2;

        for_each_valid_link(wdev, link_id) {
                if (wdev->iftype == NL80211_IFTYPE_STATION &&
                    wdev->links[link_id].client.current_bss)
                        other_chan = wdev->links[link_id].client.current_bss->pub.channel;

                /*
                 * If a GO already operates on the same GO_CONCURRENT channel,
                 * this one (maybe the same one) can beacon as well. We allow
                 * the operation even if the station we relied on with
                 * GO_CONCURRENT is disconnected now. But then we must make sure
                 * we're not outdoor on an indoor-only channel.
                 */
                if (iftype == NL80211_IFTYPE_P2P_GO &&
                    wdev->iftype == NL80211_IFTYPE_P2P_GO &&
                    wdev->links[link_id].ap.beacon_interval &&
                    !(chan->flags & IEEE80211_CHAN_INDOOR_ONLY))
                        other_chan = wdev->links[link_id].ap.chandef.chan;

                if (!other_chan)
                        continue;

                if (chan == other_chan)
                        return true;

                if (chan->band != NL80211_BAND_5GHZ &&
                    chan->band != NL80211_BAND_6GHZ)
                        continue;

                r1 = cfg80211_get_unii(chan->center_freq);
                r2 = cfg80211_get_unii(other_chan->center_freq);

                if (r1 != -EINVAL && r1 == r2) {
                        /*
                         * At some locations channels 149-165 are considered a
                         * bundle, but at other locations, e.g., Indonesia,
                         * channels 149-161 are considered a bundle while
                         * channel 165 is left out and considered to be in a
                         * different bundle. Thus, in case that there is a
                         * station interface connected to an AP on channel 165,
                         * it is assumed that channels 149-161 are allowed for
                         * GO operations. However, having a station interface
                         * connected to an AP on channels 149-161, does not
                         * allow GO operation on channel 165.
                         */
                        if (chan->center_freq == 5825 &&
                            other_chan->center_freq != 5825)
                                continue;
                        return true;
                }
        }

        return false;
}

/*
 * Check if the channel can be used under permissive conditions mandated by
 * some regulatory bodies, i.e., the channel is marked with
 * IEEE80211_CHAN_IR_CONCURRENT and there is an additional station interface
 * associated to an AP on the same channel or on the same UNII band
 * (assuming that the AP is an authorized master).
 * In addition allow operation on a channel on which indoor operation is
 * allowed, iff we are currently operating in an indoor environment.
 */
static bool cfg80211_ir_permissive_chan(struct wiphy *wiphy,
                                        enum nl80211_iftype iftype,
                                        struct ieee80211_channel *chan)
{
        struct wireless_dev *wdev;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);

        lockdep_assert_held(&rdev->wiphy.mtx);

        if (!IS_ENABLED(CONFIG_CFG80211_REG_RELAX_NO_IR) ||
            !(wiphy->regulatory_flags & REGULATORY_ENABLE_RELAX_NO_IR))
                return false;

        /* only valid for GO and TDLS off-channel (station/p2p-CL) */
        if (iftype != NL80211_IFTYPE_P2P_GO &&
            iftype != NL80211_IFTYPE_STATION &&
            iftype != NL80211_IFTYPE_P2P_CLIENT)
                return false;

        if (regulatory_indoor_allowed() &&
            (chan->flags & IEEE80211_CHAN_INDOOR_ONLY))
                return true;

        if (!(chan->flags & IEEE80211_CHAN_IR_CONCURRENT))
                return false;

        /*
         * Generally, it is possible to rely on another device/driver to allow
         * the IR concurrent relaxation, however, since the device can further
         * enforce the relaxation (by doing a similar verifications as this),
         * and thus fail the GO instantiation, consider only the interfaces of
         * the current registered device.
         */
        list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                bool ret;

                ret = cfg80211_ir_permissive_check_wdev(iftype, wdev, chan);
                if (ret)
                        return ret;
        }

        return false;
}

static bool _cfg80211_reg_can_beacon(struct wiphy *wiphy,
                                     struct cfg80211_chan_def *chandef,
                                     enum nl80211_iftype iftype,
                                     u32 prohibited_flags,
                                     u32 permitting_flags)
{
        bool res, check_radar;
        int dfs_required;

        trace_cfg80211_reg_can_beacon(wiphy, chandef, iftype,
                                      prohibited_flags,
                                      permitting_flags);

        if (!_cfg80211_chandef_usable(wiphy, chandef,
                                      IEEE80211_CHAN_DISABLED, 0))
                return false;

        dfs_required = cfg80211_chandef_dfs_required(wiphy, chandef, iftype);
        check_radar = dfs_required != 0;

        if (dfs_required > 0 &&
            cfg80211_chandef_dfs_available(wiphy, chandef)) {
                /* We can skip IEEE80211_CHAN_NO_IR if chandef dfs available */
                prohibited_flags &= ~IEEE80211_CHAN_NO_IR;
                check_radar = false;
        }

        if (check_radar &&
            !_cfg80211_chandef_usable(wiphy, chandef,
                                      IEEE80211_CHAN_RADAR, 0))
                return false;

        res = _cfg80211_chandef_usable(wiphy, chandef,
                                       prohibited_flags,
                                       permitting_flags);

        trace_cfg80211_return_bool(res);
        return res;
}

bool cfg80211_reg_check_beaconing(struct wiphy *wiphy,
                                  struct cfg80211_chan_def *chandef,
                                  struct cfg80211_beaconing_check_config *cfg)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        u32 permitting_flags = 0;
        bool check_no_ir = true;

        /*
         * Under certain conditions suggested by some regulatory bodies a
         * GO/STA can IR on channels marked with IEEE80211_NO_IR. Set this flag
         * only if such relaxations are not enabled and the conditions are not
         * met.
         */
        if (cfg->relax) {
                lockdep_assert_held(&rdev->wiphy.mtx);
                check_no_ir = !cfg80211_ir_permissive_chan(wiphy, cfg->iftype,
                                                           chandef->chan);
        }

        if (cfg->reg_power == IEEE80211_REG_VLP_AP)
                permitting_flags |= IEEE80211_CHAN_ALLOW_6GHZ_VLP_AP;

        return _cfg80211_reg_can_beacon(wiphy, chandef, cfg->iftype,
                                        check_no_ir ? IEEE80211_CHAN_NO_IR : 0,
                                        permitting_flags);
}
EXPORT_SYMBOL(cfg80211_reg_check_beaconing);

int cfg80211_set_monitor_channel(struct cfg80211_registered_device *rdev,
                                 struct net_device *dev,
                                 struct cfg80211_chan_def *chandef)
{
        if (!rdev->ops->set_monitor_channel)
                return -EOPNOTSUPP;
        if (!cfg80211_has_monitors_only(rdev))
                return -EBUSY;

        return rdev_set_monitor_channel(rdev, dev, chandef);
}

bool cfg80211_any_usable_channels(struct wiphy *wiphy,
                                  unsigned long sband_mask,
                                  u32 prohibited_flags)
{
        int idx;

        prohibited_flags |= IEEE80211_CHAN_DISABLED;

        for_each_set_bit(idx, &sband_mask, NUM_NL80211_BANDS) {
                struct ieee80211_supported_band *sband = wiphy->bands[idx];
                int chanidx;

                if (!sband)
                        continue;

                for (chanidx = 0; chanidx < sband->n_channels; chanidx++) {
                        struct ieee80211_channel *chan;

                        chan = &sband->channels[chanidx];

                        if (chan->flags & prohibited_flags)
                                continue;

                        return true;
                }
        }

        return false;
}
EXPORT_SYMBOL(cfg80211_any_usable_channels);

struct cfg80211_chan_def *wdev_chandef(struct wireless_dev *wdev,
                                       unsigned int link_id)
{
        lockdep_assert_wiphy(wdev->wiphy);

        WARN_ON(wdev->valid_links && !(wdev->valid_links & BIT(link_id)));
        WARN_ON(!wdev->valid_links && link_id > 0);

        switch (wdev->iftype) {
        case NL80211_IFTYPE_MESH_POINT:
                return &wdev->u.mesh.chandef;
        case NL80211_IFTYPE_ADHOC:
                return &wdev->u.ibss.chandef;
        case NL80211_IFTYPE_OCB:
                return &wdev->u.ocb.chandef;
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_P2P_GO:
                return &wdev->links[link_id].ap.chandef;
        default:
                return NULL;
        }
}
EXPORT_SYMBOL(wdev_chandef);















































































   11 




   11 


   11 



   11 


   11 
    7 


    5 



    5 

    5 


    5 














    5 



    5 









    5 






    5 










    5 
    5 

    5 








    6 

    6 






















































   11 

   11 



































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Create default crypto algorithm instances.
 *
 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 */

#include <crypto/internal/aead.h>
#include <linux/completion.h>
#include <linux/ctype.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/notifier.h>
#include <linux/rtnetlink.h>
#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/string.h>

#include "internal.h"

struct cryptomgr_param {
        struct rtattr *tb[CRYPTO_MAX_ATTRS + 2];

        struct {
                struct rtattr attr;
                struct crypto_attr_type data;
        } type;

        struct {
                struct rtattr attr;
                struct crypto_attr_alg data;
        } attrs[CRYPTO_MAX_ATTRS];

        char template[CRYPTO_MAX_ALG_NAME];

        struct crypto_larval *larval;

        u32 otype;
        u32 omask;
};

struct crypto_test_param {
        char driver[CRYPTO_MAX_ALG_NAME];
        char alg[CRYPTO_MAX_ALG_NAME];
        u32 type;
};

static int cryptomgr_probe(void *data)
{
        struct cryptomgr_param *param = data;
        struct crypto_template *tmpl;
        int err = -ENOENT;

        tmpl = crypto_lookup_template(param->template);
        if (!tmpl)
                goto out;

        do {
                err = tmpl->create(tmpl, param->tb);
        } while (err == -EAGAIN && !signal_pending(current));

        crypto_tmpl_put(tmpl);

out:
        param->larval->adult = ERR_PTR(err);
        param->larval->alg.cra_flags |= CRYPTO_ALG_DEAD;
        complete_all(&param->larval->completion);
        crypto_alg_put(&param->larval->alg);
        kfree(param);
        module_put_and_kthread_exit(0);
}

static int cryptomgr_schedule_probe(struct crypto_larval *larval)
{
        struct task_struct *thread;
        struct cryptomgr_param *param;
        const char *name = larval->alg.cra_name;
        const char *p;
        unsigned int len;
        int i;

        if (!try_module_get(THIS_MODULE))
                goto err;

        param = kzalloc(sizeof(*param), GFP_KERNEL);
        if (!param)
                goto err_put_module;

        for (p = name; isalnum(*p) || *p == '-' || *p == '_'; p++)
                ;

        len = p - name;
        if (!len || *p != '(')
                goto err_free_param;

        memcpy(param->template, name, len);

        i = 0;
        for (;;) {
                name = ++p;

                for (; isalnum(*p) || *p == '-' || *p == '_'; p++)
                        ;

                if (*p == '(') {
                        int recursion = 0;

                        for (;;) {
                                if (!*++p)
                                        goto err_free_param;
                                if (*p == '(')
                                        recursion++;
                                else if (*p == ')' && !recursion--)
                                        break;
                        }

                        p++;
                }

                len = p - name;
                if (!len)
                        goto err_free_param;

                param->attrs[i].attr.rta_len = sizeof(param->attrs[i]);
                param->attrs[i].attr.rta_type = CRYPTOA_ALG;
                memcpy(param->attrs[i].data.name, name, len);

                param->tb[i + 1] = &param->attrs[i].attr;
                i++;

                if (i >= CRYPTO_MAX_ATTRS)
                        goto err_free_param;

                if (*p == ')')
                        break;

                if (*p != ',')
                        goto err_free_param;
        }

        param->tb[i + 1] = NULL;

        param->type.attr.rta_len = sizeof(param->type);
        param->type.attr.rta_type = CRYPTOA_TYPE;
        param->type.data.type = larval->alg.cra_flags & ~CRYPTO_ALG_TESTED;
        param->type.data.mask = larval->mask & ~CRYPTO_ALG_TESTED;
        param->tb[0] = &param->type.attr;

        param->otype = larval->alg.cra_flags;
        param->omask = larval->mask;

        crypto_alg_get(&larval->alg);
        param->larval = larval;

        thread = kthread_run(cryptomgr_probe, param, "cryptomgr_probe");
        if (IS_ERR(thread))
                goto err_put_larval;

        return NOTIFY_STOP;

err_put_larval:
        crypto_alg_put(&larval->alg);
err_free_param:
        kfree(param);
err_put_module:
        module_put(THIS_MODULE);
err:
        return NOTIFY_OK;
}

static int cryptomgr_test(void *data)
{
        struct crypto_test_param *param = data;
        u32 type = param->type;
        int err;

        err = alg_test(param->driver, param->alg, type, CRYPTO_ALG_TESTED);

        crypto_alg_tested(param->driver, err);

        kfree(param);
        module_put_and_kthread_exit(0);
}

static int cryptomgr_schedule_test(struct crypto_alg *alg)
{
        struct task_struct *thread;
        struct crypto_test_param *param;

        if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS))
                return NOTIFY_DONE;

        if (!try_module_get(THIS_MODULE))
                goto err;

        param = kzalloc(sizeof(*param), GFP_KERNEL);
        if (!param)
                goto err_put_module;

        memcpy(param->driver, alg->cra_driver_name, sizeof(param->driver));
        memcpy(param->alg, alg->cra_name, sizeof(param->alg));
        param->type = alg->cra_flags;

        thread = kthread_run(cryptomgr_test, param, "cryptomgr_test");
        if (IS_ERR(thread))
                goto err_free_param;

        return NOTIFY_STOP;

err_free_param:
        kfree(param);
err_put_module:
        module_put(THIS_MODULE);
err:
        return NOTIFY_OK;
}

static int cryptomgr_notify(struct notifier_block *this, unsigned long msg,
                            void *data)
{
        switch (msg) {
        case CRYPTO_MSG_ALG_REQUEST:
                return cryptomgr_schedule_probe(data);
        case CRYPTO_MSG_ALG_REGISTER:
                return cryptomgr_schedule_test(data);
        case CRYPTO_MSG_ALG_LOADED:
                break;
        }

        return NOTIFY_DONE;
}

static struct notifier_block cryptomgr_notifier = {
        .notifier_call = cryptomgr_notify,
};

static int __init cryptomgr_init(void)
{
        return crypto_register_notifier(&cryptomgr_notifier);
}

static void __exit cryptomgr_exit(void)
{
        int err = crypto_unregister_notifier(&cryptomgr_notifier);
        BUG_ON(err);
}

/*
 * This is arch_initcall() so that the crypto self-tests are run on algorithms
 * registered early by subsys_initcall().  subsys_initcall() is needed for
 * generic implementations so that they're available for comparison tests when
 * other implementations are registered later by module_init().
 */
arch_initcall(cryptomgr_init);
module_exit(cryptomgr_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Crypto Algorithm Manager");




















































































































































   56 















    1 
































































































































  271 



  248 
  270 







    1 

   94 
    7 











  220 




    2 





  217 









  218 

  218 

  218 

    1 

  217 


   84 

  188 
  133 


   95 
   56 



  217 
   85 


   56 
  216 

   56 

  218 
  218 































    2 

    2 










   56 











   56 



   56 






   56 











   55 
   56 
   56 
   56 


   56 















   56 
   56 



   56 





   56 


   54 






   56 





   56 

   56 

   56 
   56 
   56 

   56 


   56 
   54 
   54 


   56 
























































































   56 







   56 


   56 

   56 


   56 



   56 









   56 

   56 















   56 



   12 

   56 



   56 


   56 






































































   56 




   56 


   45 
   56 







   56 


   56 
   56 



































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  261 







   13 



   20 


   18 
   18 

   18 


   30 



   30 
   30 
   30 



   31 
   12 
   31 
    3 


   81 









  259 

































































   68 











   68 

   68 
   68 



   68 


























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Neighbour Discovery for IPv6
 *        Linux INET6 implementation
 *
 *        Authors:
 *        Pedro Roque                <roque@di.fc.ul.pt>
 *        Mike Shaver                <shaver@ingenia.com>
 */

/*
 *        Changes:
 *
 *        Alexey I. Froloff                :        RFC6106 (DNSSL) support
 *        Pierre Ynard                        :        export userland ND options
 *                                                through netlink (RDNSS support)
 *        Lars Fenneberg                        :        fixed MTU setting on receipt
 *                                                of an RA.
 *        Janos Farkas                        :        kmalloc failure checks
 *        Alexey Kuznetsov                :        state machine reworked
 *                                                and moved to net/core.
 *        Pekka Savola                        :        RFC2461 validation
 *        YOSHIFUJI Hideaki @USAGI        :        Verify ND options properly
 */

#define pr_fmt(fmt) "ICMPv6: " fmt

#include <linux/module.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/sched.h>
#include <linux/net.h>
#include <linux/in6.h>
#include <linux/route.h>
#include <linux/init.h>
#include <linux/rcupdate.h>
#include <linux/slab.h>
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif

#include <linux/if_addr.h>
#include <linux/if_ether.h>
#include <linux/if_arp.h>
#include <linux/ipv6.h>
#include <linux/icmpv6.h>
#include <linux/jhash.h>

#include <net/sock.h>
#include <net/snmp.h>

#include <net/ipv6.h>
#include <net/protocol.h>
#include <net/ndisc.h>
#include <net/ip6_route.h>
#include <net/addrconf.h>
#include <net/icmp.h>

#include <net/netlink.h>
#include <linux/rtnetlink.h>

#include <net/flow.h>
#include <net/ip6_checksum.h>
#include <net/inet_common.h>
#include <linux/proc_fs.h>

#include <linux/netfilter.h>
#include <linux/netfilter_ipv6.h>

static u32 ndisc_hash(const void *pkey,
                      const struct net_device *dev,
                      __u32 *hash_rnd);
static bool ndisc_key_eq(const struct neighbour *neigh, const void *pkey);
static bool ndisc_allow_add(const struct net_device *dev,
                            struct netlink_ext_ack *extack);
static int ndisc_constructor(struct neighbour *neigh);
static void ndisc_solicit(struct neighbour *neigh, struct sk_buff *skb);
static void ndisc_error_report(struct neighbour *neigh, struct sk_buff *skb);
static int pndisc_constructor(struct pneigh_entry *n);
static void pndisc_destructor(struct pneigh_entry *n);
static void pndisc_redo(struct sk_buff *skb);
static int ndisc_is_multicast(const void *pkey);

static const struct neigh_ops ndisc_generic_ops = {
        .family =                AF_INET6,
        .solicit =                ndisc_solicit,
        .error_report =                ndisc_error_report,
        .output =                neigh_resolve_output,
        .connected_output =        neigh_connected_output,
};

static const struct neigh_ops ndisc_hh_ops = {
        .family =                AF_INET6,
        .solicit =                ndisc_solicit,
        .error_report =                ndisc_error_report,
        .output =                neigh_resolve_output,
        .connected_output =        neigh_resolve_output,
};


static const struct neigh_ops ndisc_direct_ops = {
        .family =                AF_INET6,
        .output =                neigh_direct_output,
        .connected_output =        neigh_direct_output,
};

struct neigh_table nd_tbl = {
        .family =        AF_INET6,
        .key_len =        sizeof(struct in6_addr),
        .protocol =        cpu_to_be16(ETH_P_IPV6),
        .hash =                ndisc_hash,
        .key_eq =        ndisc_key_eq,
        .constructor =        ndisc_constructor,
        .pconstructor =        pndisc_constructor,
        .pdestructor =        pndisc_destructor,
        .proxy_redo =        pndisc_redo,
        .is_multicast =        ndisc_is_multicast,
        .allow_add  =   ndisc_allow_add,
        .id =                "ndisc_cache",
        .parms = {
                .tbl                        = &nd_tbl,
                .reachable_time                = ND_REACHABLE_TIME,
                .data = {
                        [NEIGH_VAR_MCAST_PROBES] = 3,
                        [NEIGH_VAR_UCAST_PROBES] = 3,
                        [NEIGH_VAR_RETRANS_TIME] = ND_RETRANS_TIMER,
                        [NEIGH_VAR_BASE_REACHABLE_TIME] = ND_REACHABLE_TIME,
                        [NEIGH_VAR_DELAY_PROBE_TIME] = 5 * HZ,
                        [NEIGH_VAR_INTERVAL_PROBE_TIME_MS] = 5 * HZ,
                        [NEIGH_VAR_GC_STALETIME] = 60 * HZ,
                        [NEIGH_VAR_QUEUE_LEN_BYTES] = SK_WMEM_MAX,
                        [NEIGH_VAR_PROXY_QLEN] = 64,
                        [NEIGH_VAR_ANYCAST_DELAY] = 1 * HZ,
                        [NEIGH_VAR_PROXY_DELAY] = (8 * HZ) / 10,
                },
        },
        .gc_interval =          30 * HZ,
        .gc_thresh1 =         128,
        .gc_thresh2 =         512,
        .gc_thresh3 =        1024,
};
EXPORT_SYMBOL_GPL(nd_tbl);

void __ndisc_fill_addr_option(struct sk_buff *skb, int type, const void *data,
                              int data_len, int pad)
{
        int space = __ndisc_opt_addr_space(data_len, pad);
        u8 *opt = skb_put(skb, space);

        opt[0] = type;
        opt[1] = space>>3;

        memset(opt + 2, 0, pad);
        opt   += pad;
        space -= pad;

        memcpy(opt+2, data, data_len);
        data_len += 2;
        opt += data_len;
        space -= data_len;
        if (space > 0)
                memset(opt, 0, space);
}
EXPORT_SYMBOL_GPL(__ndisc_fill_addr_option);

static inline void ndisc_fill_addr_option(struct sk_buff *skb, int type,
                                          const void *data, u8 icmp6_type)
{
        __ndisc_fill_addr_option(skb, type, data, skb->dev->addr_len,
                                 ndisc_addr_option_pad(skb->dev->type));
        ndisc_ops_fill_addr_option(skb->dev, skb, icmp6_type);
}

static inline void ndisc_fill_redirect_addr_option(struct sk_buff *skb,
                                                   void *ha,
                                                   const u8 *ops_data)
{
        ndisc_fill_addr_option(skb, ND_OPT_TARGET_LL_ADDR, ha, NDISC_REDIRECT);
        ndisc_ops_fill_redirect_addr_option(skb->dev, skb, ops_data);
}

static struct nd_opt_hdr *ndisc_next_option(struct nd_opt_hdr *cur,
                                            struct nd_opt_hdr *end)
{
        int type;
        if (!cur || !end || cur >= end)
                return NULL;
        type = cur->nd_opt_type;
        do {
                cur = ((void *)cur) + (cur->nd_opt_len << 3);
        } while (cur < end && cur->nd_opt_type != type);
        return cur <= end && cur->nd_opt_type == type ? cur : NULL;
}

static inline int ndisc_is_useropt(const struct net_device *dev,
                                   struct nd_opt_hdr *opt)
{
        return opt->nd_opt_type == ND_OPT_PREFIX_INFO ||
                opt->nd_opt_type == ND_OPT_RDNSS ||
                opt->nd_opt_type == ND_OPT_DNSSL ||
                opt->nd_opt_type == ND_OPT_6CO ||
                opt->nd_opt_type == ND_OPT_CAPTIVE_PORTAL ||
                opt->nd_opt_type == ND_OPT_PREF64;
}

static struct nd_opt_hdr *ndisc_next_useropt(const struct net_device *dev,
                                             struct nd_opt_hdr *cur,
                                             struct nd_opt_hdr *end)
{
        if (!cur || !end || cur >= end)
                return NULL;
        do {
                cur = ((void *)cur) + (cur->nd_opt_len << 3);
        } while (cur < end && !ndisc_is_useropt(dev, cur));
        return cur <= end && ndisc_is_useropt(dev, cur) ? cur : NULL;
}

struct ndisc_options *ndisc_parse_options(const struct net_device *dev,
                                          u8 *opt, int opt_len,
                                          struct ndisc_options *ndopts)
{
        struct nd_opt_hdr *nd_opt = (struct nd_opt_hdr *)opt;

        if (!nd_opt || opt_len < 0 || !ndopts)
                return NULL;
        memset(ndopts, 0, sizeof(*ndopts));
        while (opt_len) {
                bool unknown = false;
                int l;
                if (opt_len < sizeof(struct nd_opt_hdr))
                        return NULL;
                l = nd_opt->nd_opt_len << 3;
                if (opt_len < l || l == 0)
                        return NULL;
                if (ndisc_ops_parse_options(dev, nd_opt, ndopts))
                        goto next_opt;
                switch (nd_opt->nd_opt_type) {
                case ND_OPT_SOURCE_LL_ADDR:
                case ND_OPT_TARGET_LL_ADDR:
                case ND_OPT_MTU:
                case ND_OPT_NONCE:
                case ND_OPT_REDIRECT_HDR:
                        if (ndopts->nd_opt_array[nd_opt->nd_opt_type]) {
                                ND_PRINTK(2, warn,
                                          "%s: duplicated ND6 option found: type=%d\n",
                                          __func__, nd_opt->nd_opt_type);
                        } else {
                                ndopts->nd_opt_array[nd_opt->nd_opt_type] = nd_opt;
                        }
                        break;
                case ND_OPT_PREFIX_INFO:
                        ndopts->nd_opts_pi_end = nd_opt;
                        if (!ndopts->nd_opt_array[nd_opt->nd_opt_type])
                                ndopts->nd_opt_array[nd_opt->nd_opt_type] = nd_opt;
                        break;
#ifdef CONFIG_IPV6_ROUTE_INFO
                case ND_OPT_ROUTE_INFO:
                        ndopts->nd_opts_ri_end = nd_opt;
                        if (!ndopts->nd_opts_ri)
                                ndopts->nd_opts_ri = nd_opt;
                        break;
#endif
                default:
                        unknown = true;
                }
                if (ndisc_is_useropt(dev, nd_opt)) {
                        ndopts->nd_useropts_end = nd_opt;
                        if (!ndopts->nd_useropts)
                                ndopts->nd_useropts = nd_opt;
                } else if (unknown) {
                        /*
                         * Unknown options must be silently ignored,
                         * to accommodate future extension to the
                         * protocol.
                         */
                        ND_PRINTK(2, notice,
                                  "%s: ignored unsupported option; type=%d, len=%d\n",
                                  __func__,
                                  nd_opt->nd_opt_type,
                                  nd_opt->nd_opt_len);
                }
next_opt:
                opt_len -= l;
                nd_opt = ((void *)nd_opt) + l;
        }
        return ndopts;
}

int ndisc_mc_map(const struct in6_addr *addr, char *buf, struct net_device *dev, int dir)
{
        switch (dev->type) {
        case ARPHRD_ETHER:
        case ARPHRD_IEEE802:        /* Not sure. Check it later. --ANK */
        case ARPHRD_FDDI:
                ipv6_eth_mc_map(addr, buf);
                return 0;
        case ARPHRD_ARCNET:
                ipv6_arcnet_mc_map(addr, buf);
                return 0;
        case ARPHRD_INFINIBAND:
                ipv6_ib_mc_map(addr, dev->broadcast, buf);
                return 0;
        case ARPHRD_IPGRE:
                return ipv6_ipgre_mc_map(addr, dev->broadcast, buf);
        default:
                if (dir) {
                        memcpy(buf, dev->broadcast, dev->addr_len);
                        return 0;
                }
        }
        return -EINVAL;
}
EXPORT_SYMBOL(ndisc_mc_map);

static u32 ndisc_hash(const void *pkey,
                      const struct net_device *dev,
                      __u32 *hash_rnd)
{
        return ndisc_hashfn(pkey, dev, hash_rnd);
}

static bool ndisc_key_eq(const struct neighbour *n, const void *pkey)
{
        return neigh_key_eq128(n, pkey);
}

static int ndisc_constructor(struct neighbour *neigh)
{
        struct in6_addr *addr = (struct in6_addr *)&neigh->primary_key;
        struct net_device *dev = neigh->dev;
        struct inet6_dev *in6_dev;
        struct neigh_parms *parms;
        bool is_multicast = ipv6_addr_is_multicast(addr);

        in6_dev = in6_dev_get(dev);
        if (!in6_dev) {
                return -EINVAL;
        }

        parms = in6_dev->nd_parms;
        __neigh_parms_put(neigh->parms);
        neigh->parms = neigh_parms_clone(parms);

        neigh->type = is_multicast ? RTN_MULTICAST : RTN_UNICAST;
        if (!dev->header_ops) {
                neigh->nud_state = NUD_NOARP;
                neigh->ops = &ndisc_direct_ops;
                neigh->output = neigh_direct_output;
        } else {
                if (is_multicast) {
                        neigh->nud_state = NUD_NOARP;
                        ndisc_mc_map(addr, neigh->ha, dev, 1);
                } else if (dev->flags&(IFF_NOARP|IFF_LOOPBACK)) {
                        neigh->nud_state = NUD_NOARP;
                        memcpy(neigh->ha, dev->dev_addr, dev->addr_len);
                        if (dev->flags&IFF_LOOPBACK)
                                neigh->type = RTN_LOCAL;
                } else if (dev->flags&IFF_POINTOPOINT) {
                        neigh->nud_state = NUD_NOARP;
                        memcpy(neigh->ha, dev->broadcast, dev->addr_len);
                }
                if (dev->header_ops->cache)
                        neigh->ops = &ndisc_hh_ops;
                else
                        neigh->ops = &ndisc_generic_ops;
                if (neigh->nud_state&NUD_VALID)
                        neigh->output = neigh->ops->connected_output;
                else
                        neigh->output = neigh->ops->output;
        }
        in6_dev_put(in6_dev);
        return 0;
}

static int pndisc_constructor(struct pneigh_entry *n)
{
        struct in6_addr *addr = (struct in6_addr *)&n->key;
        struct in6_addr maddr;
        struct net_device *dev = n->dev;

        if (!dev || !__in6_dev_get(dev))
                return -EINVAL;
        addrconf_addr_solict_mult(addr, &maddr);
        ipv6_dev_mc_inc(dev, &maddr);
        return 0;
}

static void pndisc_destructor(struct pneigh_entry *n)
{
        struct in6_addr *addr = (struct in6_addr *)&n->key;
        struct in6_addr maddr;
        struct net_device *dev = n->dev;

        if (!dev || !__in6_dev_get(dev))
                return;
        addrconf_addr_solict_mult(addr, &maddr);
        ipv6_dev_mc_dec(dev, &maddr);
}

/* called with rtnl held */
static bool ndisc_allow_add(const struct net_device *dev,
                            struct netlink_ext_ack *extack)
{
        struct inet6_dev *idev = __in6_dev_get(dev);

        if (!idev || idev->cnf.disable_ipv6) {
                NL_SET_ERR_MSG(extack, "IPv6 is disabled on this device");
                return false;
        }

        return true;
}

static struct sk_buff *ndisc_alloc_skb(struct net_device *dev,
                                       int len)
{
        int hlen = LL_RESERVED_SPACE(dev);
        int tlen = dev->needed_tailroom;
        struct sock *sk = dev_net(dev)->ipv6.ndisc_sk;
        struct sk_buff *skb;

        skb = alloc_skb(hlen + sizeof(struct ipv6hdr) + len + tlen, GFP_ATOMIC);
        if (!skb) {
                ND_PRINTK(0, err, "ndisc: %s failed to allocate an skb\n",
                          __func__);
                return NULL;
        }

        skb->protocol = htons(ETH_P_IPV6);
        skb->dev = dev;

        skb_reserve(skb, hlen + sizeof(struct ipv6hdr));
        skb_reset_transport_header(skb);

        /* Manually assign socket ownership as we avoid calling
         * sock_alloc_send_pskb() to bypass wmem buffer limits
         */
        skb_set_owner_w(skb, sk);

        return skb;
}

static void ip6_nd_hdr(struct sk_buff *skb,
                       const struct in6_addr *saddr,
                       const struct in6_addr *daddr,
                       int hop_limit, int len)
{
        struct ipv6hdr *hdr;
        struct inet6_dev *idev;
        unsigned tclass;

        rcu_read_lock();
        idev = __in6_dev_get(skb->dev);
        tclass = idev ? READ_ONCE(idev->cnf.ndisc_tclass) : 0;
        rcu_read_unlock();

        skb_push(skb, sizeof(*hdr));
        skb_reset_network_header(skb);
        hdr = ipv6_hdr(skb);

        ip6_flow_hdr(hdr, tclass, 0);

        hdr->payload_len = htons(len);
        hdr->nexthdr = IPPROTO_ICMPV6;
        hdr->hop_limit = hop_limit;

        hdr->saddr = *saddr;
        hdr->daddr = *daddr;
}

void ndisc_send_skb(struct sk_buff *skb, const struct in6_addr *daddr,
                    const struct in6_addr *saddr)
{
        struct dst_entry *dst = skb_dst(skb);
        struct net *net = dev_net(skb->dev);
        struct sock *sk = net->ipv6.ndisc_sk;
        struct inet6_dev *idev;
        int err;
        struct icmp6hdr *icmp6h = icmp6_hdr(skb);
        u8 type;

        type = icmp6h->icmp6_type;

        if (!dst) {
                struct flowi6 fl6;
                int oif = skb->dev->ifindex;

                icmpv6_flow_init(sk, &fl6, type, saddr, daddr, oif);
                dst = icmp6_dst_alloc(skb->dev, &fl6);
                if (IS_ERR(dst)) {
                        kfree_skb(skb);
                        return;
                }

                skb_dst_set(skb, dst);
        }

        icmp6h->icmp6_cksum = csum_ipv6_magic(saddr, daddr, skb->len,
                                              IPPROTO_ICMPV6,
                                              csum_partial(icmp6h,
                                                           skb->len, 0));

        ip6_nd_hdr(skb, saddr, daddr, READ_ONCE(inet6_sk(sk)->hop_limit), skb->len);

        rcu_read_lock();
        idev = __in6_dev_get(dst->dev);
        IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTREQUESTS);

        err = NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT,
                      net, sk, skb, NULL, dst->dev,
                      dst_output);
        if (!err) {
                ICMP6MSGOUT_INC_STATS(net, idev, type);
                ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS);
        }

        rcu_read_unlock();
}
EXPORT_SYMBOL(ndisc_send_skb);

void ndisc_send_na(struct net_device *dev, const struct in6_addr *daddr,
                   const struct in6_addr *solicited_addr,
                   bool router, bool solicited, bool override, bool inc_opt)
{
        struct sk_buff *skb;
        struct in6_addr tmpaddr;
        struct inet6_ifaddr *ifp;
        const struct in6_addr *src_addr;
        struct nd_msg *msg;
        int optlen = 0;

        /* for anycast or proxy, solicited_addr != src_addr */
        ifp = ipv6_get_ifaddr(dev_net(dev), solicited_addr, dev, 1);
        if (ifp) {
                src_addr = solicited_addr;
                if (ifp->flags & IFA_F_OPTIMISTIC)
                        override = false;
                inc_opt |= READ_ONCE(ifp->idev->cnf.force_tllao);
                in6_ifa_put(ifp);
        } else {
                if (ipv6_dev_get_saddr(dev_net(dev), dev, daddr,
                                       inet6_sk(dev_net(dev)->ipv6.ndisc_sk)->srcprefs,
                                       &tmpaddr))
                        return;
                src_addr = &tmpaddr;
        }

        if (!dev->addr_len)
                inc_opt = false;
        if (inc_opt)
                optlen += ndisc_opt_addr_space(dev,
                                               NDISC_NEIGHBOUR_ADVERTISEMENT);

        skb = ndisc_alloc_skb(dev, sizeof(*msg) + optlen);
        if (!skb)
                return;

        msg = skb_put(skb, sizeof(*msg));
        *msg = (struct nd_msg) {
                .icmph = {
                        .icmp6_type = NDISC_NEIGHBOUR_ADVERTISEMENT,
                        .icmp6_router = router,
                        .icmp6_solicited = solicited,
                        .icmp6_override = override,
                },
                .target = *solicited_addr,
        };

        if (inc_opt)
                ndisc_fill_addr_option(skb, ND_OPT_TARGET_LL_ADDR,
                                       dev->dev_addr,
                                       NDISC_NEIGHBOUR_ADVERTISEMENT);

        ndisc_send_skb(skb, daddr, src_addr);
}

static void ndisc_send_unsol_na(struct net_device *dev)
{
        struct inet6_dev *idev;
        struct inet6_ifaddr *ifa;

        idev = in6_dev_get(dev);
        if (!idev)
                return;

        read_lock_bh(&idev->lock);
        list_for_each_entry(ifa, &idev->addr_list, if_list) {
                /* skip tentative addresses until dad completes */
                if (ifa->flags & IFA_F_TENTATIVE &&
                    !(ifa->flags & IFA_F_OPTIMISTIC))
                        continue;

                ndisc_send_na(dev, &in6addr_linklocal_allnodes, &ifa->addr,
                              /*router=*/ !!idev->cnf.forwarding,
                              /*solicited=*/ false, /*override=*/ true,
                              /*inc_opt=*/ true);
        }
        read_unlock_bh(&idev->lock);

        in6_dev_put(idev);
}

struct sk_buff *ndisc_ns_create(struct net_device *dev, const struct in6_addr *solicit,
                                const struct in6_addr *saddr, u64 nonce)
{
        int inc_opt = dev->addr_len;
        struct sk_buff *skb;
        struct nd_msg *msg;
        int optlen = 0;

        if (!saddr)
                return NULL;

        if (ipv6_addr_any(saddr))
                inc_opt = false;
        if (inc_opt)
                optlen += ndisc_opt_addr_space(dev,
                                               NDISC_NEIGHBOUR_SOLICITATION);
        if (nonce != 0)
                optlen += 8;

        skb = ndisc_alloc_skb(dev, sizeof(*msg) + optlen);
        if (!skb)
                return NULL;

        msg = skb_put(skb, sizeof(*msg));
        *msg = (struct nd_msg) {
                .icmph = {
                        .icmp6_type = NDISC_NEIGHBOUR_SOLICITATION,
                },
                .target = *solicit,
        };

        if (inc_opt)
                ndisc_fill_addr_option(skb, ND_OPT_SOURCE_LL_ADDR,
                                       dev->dev_addr,
                                       NDISC_NEIGHBOUR_SOLICITATION);
        if (nonce != 0) {
                u8 *opt = skb_put(skb, 8);

                opt[0] = ND_OPT_NONCE;
                opt[1] = 8 >> 3;
                memcpy(opt + 2, &nonce, 6);
        }

        return skb;
}
EXPORT_SYMBOL(ndisc_ns_create);

void ndisc_send_ns(struct net_device *dev, const struct in6_addr *solicit,
                   const struct in6_addr *daddr, const struct in6_addr *saddr,
                   u64 nonce)
{
        struct in6_addr addr_buf;
        struct sk_buff *skb;

        if (!saddr) {
                if (ipv6_get_lladdr(dev, &addr_buf,
                                    (IFA_F_TENTATIVE | IFA_F_OPTIMISTIC)))
                        return;
                saddr = &addr_buf;
        }

        skb = ndisc_ns_create(dev, solicit, saddr, nonce);

        if (skb)
                ndisc_send_skb(skb, daddr, saddr);
}

void ndisc_send_rs(struct net_device *dev, const struct in6_addr *saddr,
                   const struct in6_addr *daddr)
{
        struct sk_buff *skb;
        struct rs_msg *msg;
        int send_sllao = dev->addr_len;
        int optlen = 0;

#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
        /*
         * According to section 2.2 of RFC 4429, we must not
         * send router solicitations with a sllao from
         * optimistic addresses, but we may send the solicitation
         * if we don't include the sllao.  So here we check
         * if our address is optimistic, and if so, we
         * suppress the inclusion of the sllao.
         */
        if (send_sllao) {
                struct inet6_ifaddr *ifp = ipv6_get_ifaddr(dev_net(dev), saddr,
                                                           dev, 1);
                if (ifp) {
                        if (ifp->flags & IFA_F_OPTIMISTIC)  {
                                send_sllao = 0;
                        }
                        in6_ifa_put(ifp);
                } else {
                        send_sllao = 0;
                }
        }
#endif
        if (send_sllao)
                optlen += ndisc_opt_addr_space(dev, NDISC_ROUTER_SOLICITATION);

        skb = ndisc_alloc_skb(dev, sizeof(*msg) + optlen);
        if (!skb)
                return;

        msg = skb_put(skb, sizeof(*msg));
        *msg = (struct rs_msg) {
                .icmph = {
                        .icmp6_type = NDISC_ROUTER_SOLICITATION,
                },
        };

        if (send_sllao)
                ndisc_fill_addr_option(skb, ND_OPT_SOURCE_LL_ADDR,
                                       dev->dev_addr,
                                       NDISC_ROUTER_SOLICITATION);

        ndisc_send_skb(skb, daddr, saddr);
}


static void ndisc_error_report(struct neighbour *neigh, struct sk_buff *skb)
{
        /*
         *        "The sender MUST return an ICMP
         *         destination unreachable"
         */
        dst_link_failure(skb);
        kfree_skb(skb);
}

/* Called with locked neigh: either read or both */

static void ndisc_solicit(struct neighbour *neigh, struct sk_buff *skb)
{
        struct in6_addr *saddr = NULL;
        struct in6_addr mcaddr;
        struct net_device *dev = neigh->dev;
        struct in6_addr *target = (struct in6_addr *)&neigh->primary_key;
        int probes = atomic_read(&neigh->probes);

        if (skb && ipv6_chk_addr_and_flags(dev_net(dev), &ipv6_hdr(skb)->saddr,
                                           dev, false, 1,
                                           IFA_F_TENTATIVE|IFA_F_OPTIMISTIC))
                saddr = &ipv6_hdr(skb)->saddr;
        probes -= NEIGH_VAR(neigh->parms, UCAST_PROBES);
        if (probes < 0) {
                if (!(READ_ONCE(neigh->nud_state) & NUD_VALID)) {
                        ND_PRINTK(1, dbg,
                                  "%s: trying to ucast probe in NUD_INVALID: %pI6\n",
                                  __func__, target);
                }
                ndisc_send_ns(dev, target, target, saddr, 0);
        } else if ((probes -= NEIGH_VAR(neigh->parms, APP_PROBES)) < 0) {
                neigh_app_ns(neigh);
        } else {
                addrconf_addr_solict_mult(target, &mcaddr);
                ndisc_send_ns(dev, target, &mcaddr, saddr, 0);
        }
}

static int pndisc_is_router(const void *pkey,
                            struct net_device *dev)
{
        struct pneigh_entry *n;
        int ret = -1;

        read_lock_bh(&nd_tbl.lock);
        n = __pneigh_lookup(&nd_tbl, dev_net(dev), pkey, dev);
        if (n)
                ret = !!(n->flags & NTF_ROUTER);
        read_unlock_bh(&nd_tbl.lock);

        return ret;
}

void ndisc_update(const struct net_device *dev, struct neighbour *neigh,
                  const u8 *lladdr, u8 new, u32 flags, u8 icmp6_type,
                  struct ndisc_options *ndopts)
{
        neigh_update(neigh, lladdr, new, flags, 0);
        /* report ndisc ops about neighbour update */
        ndisc_ops_update(dev, neigh, flags, icmp6_type, ndopts);
}

static enum skb_drop_reason ndisc_recv_ns(struct sk_buff *skb)
{
        struct nd_msg *msg = (struct nd_msg *)skb_transport_header(skb);
        const struct in6_addr *saddr = &ipv6_hdr(skb)->saddr;
        const struct in6_addr *daddr = &ipv6_hdr(skb)->daddr;
        u8 *lladdr = NULL;
        u32 ndoptlen = skb_tail_pointer(skb) - (skb_transport_header(skb) +
                                    offsetof(struct nd_msg, opt));
        struct ndisc_options ndopts;
        struct net_device *dev = skb->dev;
        struct inet6_ifaddr *ifp;
        struct inet6_dev *idev = NULL;
        struct neighbour *neigh;
        int dad = ipv6_addr_any(saddr);
        int is_router = -1;
        SKB_DR(reason);
        u64 nonce = 0;
        bool inc;

        if (skb->len < sizeof(struct nd_msg))
                return SKB_DROP_REASON_PKT_TOO_SMALL;

        if (ipv6_addr_is_multicast(&msg->target)) {
                ND_PRINTK(2, warn, "NS: multicast target address\n");
                return reason;
        }

        /*
         * RFC2461 7.1.1:
         * DAD has to be destined for solicited node multicast address.
         */
        if (dad && !ipv6_addr_is_solict_mult(daddr)) {
                ND_PRINTK(2, warn, "NS: bad DAD packet (wrong destination)\n");
                return reason;
        }

        if (!ndisc_parse_options(dev, msg->opt, ndoptlen, &ndopts))
                return SKB_DROP_REASON_IPV6_NDISC_BAD_OPTIONS;

        if (ndopts.nd_opts_src_lladdr) {
                lladdr = ndisc_opt_addr_data(ndopts.nd_opts_src_lladdr, dev);
                if (!lladdr) {
                        ND_PRINTK(2, warn,
                                  "NS: invalid link-layer address length\n");
                        return reason;
                }

                /* RFC2461 7.1.1:
                 *        If the IP source address is the unspecified address,
                 *        there MUST NOT be source link-layer address option
                 *        in the message.
                 */
                if (dad) {
                        ND_PRINTK(2, warn,
                                  "NS: bad DAD packet (link-layer address option)\n");
                        return reason;
                }
        }
        if (ndopts.nd_opts_nonce && ndopts.nd_opts_nonce->nd_opt_len == 1)
                memcpy(&nonce, (u8 *)(ndopts.nd_opts_nonce + 1), 6);

        inc = ipv6_addr_is_multicast(daddr);

        ifp = ipv6_get_ifaddr(dev_net(dev), &msg->target, dev, 1);
        if (ifp) {
have_ifp:
                if (ifp->flags & (IFA_F_TENTATIVE|IFA_F_OPTIMISTIC)) {
                        if (dad) {
                                if (nonce != 0 && ifp->dad_nonce == nonce) {
                                        u8 *np = (u8 *)&nonce;
                                        /* Matching nonce if looped back */
                                        ND_PRINTK(2, notice,
                                                  "%s: IPv6 DAD loopback for address %pI6c nonce %pM ignored\n",
                                                  ifp->idev->dev->name,
                                                  &ifp->addr, np);
                                        goto out;
                                }
                                /*
                                 * We are colliding with another node
                                 * who is doing DAD
                                 * so fail our DAD process
                                 */
                                addrconf_dad_failure(skb, ifp);
                                return reason;
                        } else {
                                /*
                                 * This is not a dad solicitation.
                                 * If we are an optimistic node,
                                 * we should respond.
                                 * Otherwise, we should ignore it.
                                 */
                                if (!(ifp->flags & IFA_F_OPTIMISTIC))
                                        goto out;
                        }
                }

                idev = ifp->idev;
        } else {
                struct net *net = dev_net(dev);

                /* perhaps an address on the master device */
                if (netif_is_l3_slave(dev)) {
                        struct net_device *mdev;

                        mdev = netdev_master_upper_dev_get_rcu(dev);
                        if (mdev) {
                                ifp = ipv6_get_ifaddr(net, &msg->target, mdev, 1);
                                if (ifp)
                                        goto have_ifp;
                        }
                }

                idev = in6_dev_get(dev);
                if (!idev) {
                        /* XXX: count this drop? */
                        return reason;
                }

                if (ipv6_chk_acast_addr(net, dev, &msg->target) ||
                    (READ_ONCE(idev->cnf.forwarding) &&
                     (READ_ONCE(net->ipv6.devconf_all->proxy_ndp) ||
                      READ_ONCE(idev->cnf.proxy_ndp)) &&
                     (is_router = pndisc_is_router(&msg->target, dev)) >= 0)) {
                        if (!(NEIGH_CB(skb)->flags & LOCALLY_ENQUEUED) &&
                            skb->pkt_type != PACKET_HOST &&
                            inc &&
                            NEIGH_VAR(idev->nd_parms, PROXY_DELAY) != 0) {
                                /*
                                 * for anycast or proxy,
                                 * sender should delay its response
                                 * by a random time between 0 and
                                 * MAX_ANYCAST_DELAY_TIME seconds.
                                 * (RFC2461) -- yoshfuji
                                 */
                                struct sk_buff *n = skb_clone(skb, GFP_ATOMIC);
                                if (n)
                                        pneigh_enqueue(&nd_tbl, idev->nd_parms, n);
                                goto out;
                        }
                } else {
                        SKB_DR_SET(reason, IPV6_NDISC_NS_OTHERHOST);
                        goto out;
                }
        }

        if (is_router < 0)
                is_router = READ_ONCE(idev->cnf.forwarding);

        if (dad) {
                ndisc_send_na(dev, &in6addr_linklocal_allnodes, &msg->target,
                              !!is_router, false, (ifp != NULL), true);
                goto out;
        }

        if (inc)
                NEIGH_CACHE_STAT_INC(&nd_tbl, rcv_probes_mcast);
        else
                NEIGH_CACHE_STAT_INC(&nd_tbl, rcv_probes_ucast);

        /*
         *        update / create cache entry
         *        for the source address
         */
        neigh = __neigh_lookup(&nd_tbl, saddr, dev,
                               !inc || lladdr || !dev->addr_len);
        if (neigh)
                ndisc_update(dev, neigh, lladdr, NUD_STALE,
                             NEIGH_UPDATE_F_WEAK_OVERRIDE|
                             NEIGH_UPDATE_F_OVERRIDE,
                             NDISC_NEIGHBOUR_SOLICITATION, &ndopts);
        if (neigh || !dev->header_ops) {
                ndisc_send_na(dev, saddr, &msg->target, !!is_router,
                              true, (ifp != NULL && inc), inc);
                if (neigh)
                        neigh_release(neigh);
                reason = SKB_CONSUMED;
        }

out:
        if (ifp)
                in6_ifa_put(ifp);
        else
                in6_dev_put(idev);
        return reason;
}

static int accept_untracked_na(struct net_device *dev, struct in6_addr *saddr)
{
        struct inet6_dev *idev = __in6_dev_get(dev);

        switch (READ_ONCE(idev->cnf.accept_untracked_na)) {
        case 0: /* Don't accept untracked na (absent in neighbor cache) */
                return 0;
        case 1: /* Create new entries from na if currently untracked */
                return 1;
        case 2: /* Create new entries from untracked na only if saddr is in the
                 * same subnet as an address configured on the interface that
                 * received the na
                 */
                return !!ipv6_chk_prefix(saddr, dev);
        default:
                return 0;
        }
}

static enum skb_drop_reason ndisc_recv_na(struct sk_buff *skb)
{
        struct nd_msg *msg = (struct nd_msg *)skb_transport_header(skb);
        struct in6_addr *saddr = &ipv6_hdr(skb)->saddr;
        const struct in6_addr *daddr = &ipv6_hdr(skb)->daddr;
        u8 *lladdr = NULL;
        u32 ndoptlen = skb_tail_pointer(skb) - (skb_transport_header(skb) +
                                    offsetof(struct nd_msg, opt));
        struct ndisc_options ndopts;
        struct net_device *dev = skb->dev;
        struct inet6_dev *idev = __in6_dev_get(dev);
        struct inet6_ifaddr *ifp;
        struct neighbour *neigh;
        SKB_DR(reason);
        u8 new_state;

        if (skb->len < sizeof(struct nd_msg))
                return SKB_DROP_REASON_PKT_TOO_SMALL;

        if (ipv6_addr_is_multicast(&msg->target)) {
                ND_PRINTK(2, warn, "NA: target address is multicast\n");
                return reason;
        }

        if (ipv6_addr_is_multicast(daddr) &&
            msg->icmph.icmp6_solicited) {
                ND_PRINTK(2, warn, "NA: solicited NA is multicasted\n");
                return reason;
        }

        /* For some 802.11 wireless deployments (and possibly other networks),
         * there will be a NA proxy and unsolicitd packets are attacks
         * and thus should not be accepted.
         * drop_unsolicited_na takes precedence over accept_untracked_na
         */
        if (!msg->icmph.icmp6_solicited && idev &&
            READ_ONCE(idev->cnf.drop_unsolicited_na))
                return reason;

        if (!ndisc_parse_options(dev, msg->opt, ndoptlen, &ndopts))
                return SKB_DROP_REASON_IPV6_NDISC_BAD_OPTIONS;

        if (ndopts.nd_opts_tgt_lladdr) {
                lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr, dev);
                if (!lladdr) {
                        ND_PRINTK(2, warn,
                                  "NA: invalid link-layer address length\n");
                        return reason;
                }
        }
        ifp = ipv6_get_ifaddr(dev_net(dev), &msg->target, dev, 1);
        if (ifp) {
                if (skb->pkt_type != PACKET_LOOPBACK
                    && (ifp->flags & IFA_F_TENTATIVE)) {
                                addrconf_dad_failure(skb, ifp);
                                return reason;
                }
                /* What should we make now? The advertisement
                   is invalid, but ndisc specs say nothing
                   about it. It could be misconfiguration, or
                   an smart proxy agent tries to help us :-)

                   We should not print the error if NA has been
                   received from loopback - it is just our own
                   unsolicited advertisement.
                 */
                if (skb->pkt_type != PACKET_LOOPBACK)
                        ND_PRINTK(1, warn,
                                  "NA: %pM advertised our address %pI6c on %s!\n",
                                  eth_hdr(skb)->h_source, &ifp->addr, ifp->idev->dev->name);
                in6_ifa_put(ifp);
                return reason;
        }

        neigh = neigh_lookup(&nd_tbl, &msg->target, dev);

        /* RFC 9131 updates original Neighbour Discovery RFC 4861.
         * NAs with Target LL Address option without a corresponding
         * entry in the neighbour cache can now create a STALE neighbour
         * cache entry on routers.
         *
         *   entry accept  fwding  solicited        behaviour
         * ------- ------  ------  ---------    ----------------------
         * present      X       X         0     Set state to STALE
         * present      X       X         1     Set state to REACHABLE
         *  absent      0       X         X     Do nothing
         *  absent      1       0         X     Do nothing
         *  absent      1       1         X     Add a new STALE entry
         *
         * Note that we don't do a (daddr == all-routers-mcast) check.
         */
        new_state = msg->icmph.icmp6_solicited ? NUD_REACHABLE : NUD_STALE;
        if (!neigh && lladdr && idev && READ_ONCE(idev->cnf.forwarding)) {
                if (accept_untracked_na(dev, saddr)) {
                        neigh = neigh_create(&nd_tbl, &msg->target, dev);
                        new_state = NUD_STALE;
                }
        }

        if (neigh && !IS_ERR(neigh)) {
                u8 old_flags = neigh->flags;
                struct net *net = dev_net(dev);

                if (READ_ONCE(neigh->nud_state) & NUD_FAILED)
                        goto out;

                /*
                 * Don't update the neighbor cache entry on a proxy NA from
                 * ourselves because either the proxied node is off link or it
                 * has already sent a NA to us.
                 */
                if (lladdr && !memcmp(lladdr, dev->dev_addr, dev->addr_len) &&
                    READ_ONCE(net->ipv6.devconf_all->forwarding) &&
                    READ_ONCE(net->ipv6.devconf_all->proxy_ndp) &&
                    pneigh_lookup(&nd_tbl, net, &msg->target, dev, 0)) {
                        /* XXX: idev->cnf.proxy_ndp */
                        goto out;
                }

                ndisc_update(dev, neigh, lladdr,
                             new_state,
                             NEIGH_UPDATE_F_WEAK_OVERRIDE|
                             (msg->icmph.icmp6_override ? NEIGH_UPDATE_F_OVERRIDE : 0)|
                             NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
                             (msg->icmph.icmp6_router ? NEIGH_UPDATE_F_ISROUTER : 0),
                             NDISC_NEIGHBOUR_ADVERTISEMENT, &ndopts);

                if ((old_flags & ~neigh->flags) & NTF_ROUTER) {
                        /*
                         * Change: router to host
                         */
                        rt6_clean_tohost(dev_net(dev),  saddr);
                }
                reason = SKB_CONSUMED;
out:
                neigh_release(neigh);
        }
        return reason;
}

static enum skb_drop_reason ndisc_recv_rs(struct sk_buff *skb)
{
        struct rs_msg *rs_msg = (struct rs_msg *)skb_transport_header(skb);
        unsigned long ndoptlen = skb->len - sizeof(*rs_msg);
        struct neighbour *neigh;
        struct inet6_dev *idev;
        const struct in6_addr *saddr = &ipv6_hdr(skb)->saddr;
        struct ndisc_options ndopts;
        u8 *lladdr = NULL;
        SKB_DR(reason);

        if (skb->len < sizeof(*rs_msg))
                return SKB_DROP_REASON_PKT_TOO_SMALL;

        idev = __in6_dev_get(skb->dev);
        if (!idev) {
                ND_PRINTK(1, err, "RS: can't find in6 device\n");
                return reason;
        }

        /* Don't accept RS if we're not in router mode */
        if (!READ_ONCE(idev->cnf.forwarding))
                goto out;

        /*
         * Don't update NCE if src = ::;
         * this implies that the source node has no ip address assigned yet.
         */
        if (ipv6_addr_any(saddr))
                goto out;

        /* Parse ND options */
        if (!ndisc_parse_options(skb->dev, rs_msg->opt, ndoptlen, &ndopts))
                return SKB_DROP_REASON_IPV6_NDISC_BAD_OPTIONS;

        if (ndopts.nd_opts_src_lladdr) {
                lladdr = ndisc_opt_addr_data(ndopts.nd_opts_src_lladdr,
                                             skb->dev);
                if (!lladdr)
                        goto out;
        }

        neigh = __neigh_lookup(&nd_tbl, saddr, skb->dev, 1);
        if (neigh) {
                ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
                             NEIGH_UPDATE_F_WEAK_OVERRIDE|
                             NEIGH_UPDATE_F_OVERRIDE|
                             NEIGH_UPDATE_F_OVERRIDE_ISROUTER,
                             NDISC_ROUTER_SOLICITATION, &ndopts);
                neigh_release(neigh);
                reason = SKB_CONSUMED;
        }
out:
        return reason;
}

static void ndisc_ra_useropt(struct sk_buff *ra, struct nd_opt_hdr *opt)
{
        struct icmp6hdr *icmp6h = (struct icmp6hdr *)skb_transport_header(ra);
        struct sk_buff *skb;
        struct nlmsghdr *nlh;
        struct nduseroptmsg *ndmsg;
        struct net *net = dev_net(ra->dev);
        int err;
        int base_size = NLMSG_ALIGN(sizeof(struct nduseroptmsg)
                                    + (opt->nd_opt_len << 3));
        size_t msg_size = base_size + nla_total_size(sizeof(struct in6_addr));

        skb = nlmsg_new(msg_size, GFP_ATOMIC);
        if (!skb) {
                err = -ENOBUFS;
                goto errout;
        }

        nlh = nlmsg_put(skb, 0, 0, RTM_NEWNDUSEROPT, base_size, 0);
        if (!nlh) {
                goto nla_put_failure;
        }

        ndmsg = nlmsg_data(nlh);
        ndmsg->nduseropt_family = AF_INET6;
        ndmsg->nduseropt_ifindex = ra->dev->ifindex;
        ndmsg->nduseropt_icmp_type = icmp6h->icmp6_type;
        ndmsg->nduseropt_icmp_code = icmp6h->icmp6_code;
        ndmsg->nduseropt_opts_len = opt->nd_opt_len << 3;

        memcpy(ndmsg + 1, opt, opt->nd_opt_len << 3);

        if (nla_put_in6_addr(skb, NDUSEROPT_SRCADDR, &ipv6_hdr(ra)->saddr))
                goto nla_put_failure;
        nlmsg_end(skb, nlh);

        rtnl_notify(skb, net, 0, RTNLGRP_ND_USEROPT, NULL, GFP_ATOMIC);
        return;

nla_put_failure:
        nlmsg_free(skb);
        err = -EMSGSIZE;
errout:
        rtnl_set_sk_err(net, RTNLGRP_ND_USEROPT, err);
}

static enum skb_drop_reason ndisc_router_discovery(struct sk_buff *skb)
{
        struct ra_msg *ra_msg = (struct ra_msg *)skb_transport_header(skb);
        bool send_ifinfo_notify = false;
        struct neighbour *neigh = NULL;
        struct ndisc_options ndopts;
        struct fib6_info *rt = NULL;
        struct inet6_dev *in6_dev;
        struct fib6_table *table;
        u32 defrtr_usr_metric;
        unsigned int pref = 0;
        __u32 old_if_flags;
        struct net *net;
        SKB_DR(reason);
        int lifetime;
        int optlen;

        __u8 *opt = (__u8 *)(ra_msg + 1);

        optlen = (skb_tail_pointer(skb) - skb_transport_header(skb)) -
                sizeof(struct ra_msg);

        ND_PRINTK(2, info,
                  "RA: %s, dev: %s\n",
                  __func__, skb->dev->name);
        if (!(ipv6_addr_type(&ipv6_hdr(skb)->saddr) & IPV6_ADDR_LINKLOCAL)) {
                ND_PRINTK(2, warn, "RA: source address is not link-local\n");
                return reason;
        }
        if (optlen < 0)
                return SKB_DROP_REASON_PKT_TOO_SMALL;

#ifdef CONFIG_IPV6_NDISC_NODETYPE
        if (skb->ndisc_nodetype == NDISC_NODETYPE_HOST) {
                ND_PRINTK(2, warn, "RA: from host or unauthorized router\n");
                return reason;
        }
#endif

        in6_dev = __in6_dev_get(skb->dev);
        if (!in6_dev) {
                ND_PRINTK(0, err, "RA: can't find inet6 device for %s\n",
                          skb->dev->name);
                return reason;
        }

        if (!ndisc_parse_options(skb->dev, opt, optlen, &ndopts))
                return SKB_DROP_REASON_IPV6_NDISC_BAD_OPTIONS;

        if (!ipv6_accept_ra(in6_dev)) {
                ND_PRINTK(2, info,
                          "RA: %s, did not accept ra for dev: %s\n",
                          __func__, skb->dev->name);
                goto skip_linkparms;
        }

#ifdef CONFIG_IPV6_NDISC_NODETYPE
        /* skip link-specific parameters from interior routers */
        if (skb->ndisc_nodetype == NDISC_NODETYPE_NODEFAULT) {
                ND_PRINTK(2, info,
                          "RA: %s, nodetype is NODEFAULT, dev: %s\n",
                          __func__, skb->dev->name);
                goto skip_linkparms;
        }
#endif

        if (in6_dev->if_flags & IF_RS_SENT) {
                /*
                 *        flag that an RA was received after an RS was sent
                 *        out on this interface.
                 */
                in6_dev->if_flags |= IF_RA_RCVD;
        }

        /*
         * Remember the managed/otherconf flags from most recently
         * received RA message (RFC 2462) -- yoshfuji
         */
        old_if_flags = in6_dev->if_flags;
        in6_dev->if_flags = (in6_dev->if_flags & ~(IF_RA_MANAGED |
                                IF_RA_OTHERCONF)) |
                                (ra_msg->icmph.icmp6_addrconf_managed ?
                                        IF_RA_MANAGED : 0) |
                                (ra_msg->icmph.icmp6_addrconf_other ?
                                        IF_RA_OTHERCONF : 0);

        if (old_if_flags != in6_dev->if_flags)
                send_ifinfo_notify = true;

        if (!READ_ONCE(in6_dev->cnf.accept_ra_defrtr)) {
                ND_PRINTK(2, info,
                          "RA: %s, defrtr is false for dev: %s\n",
                          __func__, skb->dev->name);
                goto skip_defrtr;
        }

        lifetime = ntohs(ra_msg->icmph.icmp6_rt_lifetime);
        if (lifetime != 0 &&
            lifetime < READ_ONCE(in6_dev->cnf.accept_ra_min_lft)) {
                ND_PRINTK(2, info,
                          "RA: router lifetime (%ds) is too short: %s\n",
                          lifetime, skb->dev->name);
                goto skip_defrtr;
        }

        /* Do not accept RA with source-addr found on local machine unless
         * accept_ra_from_local is set to true.
         */
        net = dev_net(in6_dev->dev);
        if (!READ_ONCE(in6_dev->cnf.accept_ra_from_local) &&
            ipv6_chk_addr(net, &ipv6_hdr(skb)->saddr, in6_dev->dev, 0)) {
                ND_PRINTK(2, info,
                          "RA from local address detected on dev: %s: default router ignored\n",
                          skb->dev->name);
                goto skip_defrtr;
        }

#ifdef CONFIG_IPV6_ROUTER_PREF
        pref = ra_msg->icmph.icmp6_router_pref;
        /* 10b is handled as if it were 00b (medium) */
        if (pref == ICMPV6_ROUTER_PREF_INVALID ||
            !READ_ONCE(in6_dev->cnf.accept_ra_rtr_pref))
                pref = ICMPV6_ROUTER_PREF_MEDIUM;
#endif
        /* routes added from RAs do not use nexthop objects */
        rt = rt6_get_dflt_router(net, &ipv6_hdr(skb)->saddr, skb->dev);
        if (rt) {
                neigh = ip6_neigh_lookup(&rt->fib6_nh->fib_nh_gw6,
                                         rt->fib6_nh->fib_nh_dev, NULL,
                                          &ipv6_hdr(skb)->saddr);
                if (!neigh) {
                        ND_PRINTK(0, err,
                                  "RA: %s got default router without neighbour\n",
                                  __func__);
                        fib6_info_release(rt);
                        return reason;
                }
        }
        /* Set default route metric as specified by user */
        defrtr_usr_metric = in6_dev->cnf.ra_defrtr_metric;
        /* delete the route if lifetime is 0 or if metric needs change */
        if (rt && (lifetime == 0 || rt->fib6_metric != defrtr_usr_metric)) {
                ip6_del_rt(net, rt, false);
                rt = NULL;
        }

        ND_PRINTK(3, info, "RA: rt: %p  lifetime: %d, metric: %d, for dev: %s\n",
                  rt, lifetime, defrtr_usr_metric, skb->dev->name);
        if (!rt && lifetime) {
                ND_PRINTK(3, info, "RA: adding default router\n");

                if (neigh)
                        neigh_release(neigh);

                rt = rt6_add_dflt_router(net, &ipv6_hdr(skb)->saddr,
                                         skb->dev, pref, defrtr_usr_metric,
                                         lifetime);
                if (!rt) {
                        ND_PRINTK(0, err,
                                  "RA: %s failed to add default route\n",
                                  __func__);
                        return reason;
                }

                neigh = ip6_neigh_lookup(&rt->fib6_nh->fib_nh_gw6,
                                         rt->fib6_nh->fib_nh_dev, NULL,
                                          &ipv6_hdr(skb)->saddr);
                if (!neigh) {
                        ND_PRINTK(0, err,
                                  "RA: %s got default router without neighbour\n",
                                  __func__);
                        fib6_info_release(rt);
                        return reason;
                }
                neigh->flags |= NTF_ROUTER;
        } else if (rt && IPV6_EXTRACT_PREF(rt->fib6_flags) != pref) {
                struct nl_info nlinfo = {
                        .nl_net = net,
                };
                rt->fib6_flags = (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref);
                inet6_rt_notify(RTM_NEWROUTE, rt, &nlinfo, NLM_F_REPLACE);
        }

        if (rt) {
                table = rt->fib6_table;
                spin_lock_bh(&table->tb6_lock);

                fib6_set_expires(rt, jiffies + (HZ * lifetime));
                fib6_add_gc_list(rt);

                spin_unlock_bh(&table->tb6_lock);
        }
        if (READ_ONCE(in6_dev->cnf.accept_ra_min_hop_limit) < 256 &&
            ra_msg->icmph.icmp6_hop_limit) {
                if (READ_ONCE(in6_dev->cnf.accept_ra_min_hop_limit) <=
                    ra_msg->icmph.icmp6_hop_limit) {
                        WRITE_ONCE(in6_dev->cnf.hop_limit,
                                   ra_msg->icmph.icmp6_hop_limit);
                        fib6_metric_set(rt, RTAX_HOPLIMIT,
                                        ra_msg->icmph.icmp6_hop_limit);
                } else {
                        ND_PRINTK(2, warn, "RA: Got route advertisement with lower hop_limit than minimum\n");
                }
        }

skip_defrtr:

        /*
         *        Update Reachable Time and Retrans Timer
         */

        if (in6_dev->nd_parms) {
                unsigned long rtime = ntohl(ra_msg->retrans_timer);

                if (rtime && rtime/1000 < MAX_SCHEDULE_TIMEOUT/HZ) {
                        rtime = (rtime*HZ)/1000;
                        if (rtime < HZ/100)
                                rtime = HZ/100;
                        NEIGH_VAR_SET(in6_dev->nd_parms, RETRANS_TIME, rtime);
                        in6_dev->tstamp = jiffies;
                        send_ifinfo_notify = true;
                }

                rtime = ntohl(ra_msg->reachable_time);
                if (rtime && rtime/1000 < MAX_SCHEDULE_TIMEOUT/(3*HZ)) {
                        rtime = (rtime*HZ)/1000;

                        if (rtime < HZ/10)
                                rtime = HZ/10;

                        if (rtime != NEIGH_VAR(in6_dev->nd_parms, BASE_REACHABLE_TIME)) {
                                NEIGH_VAR_SET(in6_dev->nd_parms,
                                              BASE_REACHABLE_TIME, rtime);
                                NEIGH_VAR_SET(in6_dev->nd_parms,
                                              GC_STALETIME, 3 * rtime);
                                in6_dev->nd_parms->reachable_time = neigh_rand_reach_time(rtime);
                                in6_dev->tstamp = jiffies;
                                send_ifinfo_notify = true;
                        }
                }
        }

skip_linkparms:

        /*
         *        Process options.
         */

        if (!neigh)
                neigh = __neigh_lookup(&nd_tbl, &ipv6_hdr(skb)->saddr,
                                       skb->dev, 1);
        if (neigh) {
                u8 *lladdr = NULL;
                if (ndopts.nd_opts_src_lladdr) {
                        lladdr = ndisc_opt_addr_data(ndopts.nd_opts_src_lladdr,
                                                     skb->dev);
                        if (!lladdr) {
                                ND_PRINTK(2, warn,
                                          "RA: invalid link-layer address length\n");
                                goto out;
                        }
                }
                ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
                             NEIGH_UPDATE_F_WEAK_OVERRIDE|
                             NEIGH_UPDATE_F_OVERRIDE|
                             NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
                             NEIGH_UPDATE_F_ISROUTER,
                             NDISC_ROUTER_ADVERTISEMENT, &ndopts);
                reason = SKB_CONSUMED;
        }

        if (!ipv6_accept_ra(in6_dev)) {
                ND_PRINTK(2, info,
                          "RA: %s, accept_ra is false for dev: %s\n",
                          __func__, skb->dev->name);
                goto out;
        }

#ifdef CONFIG_IPV6_ROUTE_INFO
        if (!READ_ONCE(in6_dev->cnf.accept_ra_from_local) &&
            ipv6_chk_addr(dev_net(in6_dev->dev), &ipv6_hdr(skb)->saddr,
                          in6_dev->dev, 0)) {
                ND_PRINTK(2, info,
                          "RA from local address detected on dev: %s: router info ignored.\n",
                          skb->dev->name);
                goto skip_routeinfo;
        }

        if (READ_ONCE(in6_dev->cnf.accept_ra_rtr_pref) && ndopts.nd_opts_ri) {
                struct nd_opt_hdr *p;
                for (p = ndopts.nd_opts_ri;
                     p;
                     p = ndisc_next_option(p, ndopts.nd_opts_ri_end)) {
                        struct route_info *ri = (struct route_info *)p;
#ifdef CONFIG_IPV6_NDISC_NODETYPE
                        if (skb->ndisc_nodetype == NDISC_NODETYPE_NODEFAULT &&
                            ri->prefix_len == 0)
                                continue;
#endif
                        if (ri->prefix_len == 0 &&
                            !READ_ONCE(in6_dev->cnf.accept_ra_defrtr))
                                continue;
                        if (ri->lifetime != 0 &&
                            ntohl(ri->lifetime) < READ_ONCE(in6_dev->cnf.accept_ra_min_lft))
                                continue;
                        if (ri->prefix_len < READ_ONCE(in6_dev->cnf.accept_ra_rt_info_min_plen))
                                continue;
                        if (ri->prefix_len > READ_ONCE(in6_dev->cnf.accept_ra_rt_info_max_plen))
                                continue;
                        rt6_route_rcv(skb->dev, (u8 *)p, (p->nd_opt_len) << 3,
                                      &ipv6_hdr(skb)->saddr);
                }
        }

skip_routeinfo:
#endif

#ifdef CONFIG_IPV6_NDISC_NODETYPE
        /* skip link-specific ndopts from interior routers */
        if (skb->ndisc_nodetype == NDISC_NODETYPE_NODEFAULT) {
                ND_PRINTK(2, info,
                          "RA: %s, nodetype is NODEFAULT (interior routes), dev: %s\n",
                          __func__, skb->dev->name);
                goto out;
        }
#endif

        if (READ_ONCE(in6_dev->cnf.accept_ra_pinfo) && ndopts.nd_opts_pi) {
                struct nd_opt_hdr *p;
                for (p = ndopts.nd_opts_pi;
                     p;
                     p = ndisc_next_option(p, ndopts.nd_opts_pi_end)) {
                        addrconf_prefix_rcv(skb->dev, (u8 *)p,
                                            (p->nd_opt_len) << 3,
                                            ndopts.nd_opts_src_lladdr != NULL);
                }
        }

        if (ndopts.nd_opts_mtu && READ_ONCE(in6_dev->cnf.accept_ra_mtu)) {
                __be32 n;
                u32 mtu;

                memcpy(&n, ((u8 *)(ndopts.nd_opts_mtu+1))+2, sizeof(mtu));
                mtu = ntohl(n);

                if (in6_dev->ra_mtu != mtu) {
                        in6_dev->ra_mtu = mtu;
                        send_ifinfo_notify = true;
                }

                if (mtu < IPV6_MIN_MTU || mtu > skb->dev->mtu) {
                        ND_PRINTK(2, warn, "RA: invalid mtu: %d\n", mtu);
                } else if (READ_ONCE(in6_dev->cnf.mtu6) != mtu) {
                        WRITE_ONCE(in6_dev->cnf.mtu6, mtu);
                        fib6_metric_set(rt, RTAX_MTU, mtu);
                        rt6_mtu_change(skb->dev, mtu);
                }
        }

        if (ndopts.nd_useropts) {
                struct nd_opt_hdr *p;
                for (p = ndopts.nd_useropts;
                     p;
                     p = ndisc_next_useropt(skb->dev, p,
                                            ndopts.nd_useropts_end)) {
                        ndisc_ra_useropt(skb, p);
                }
        }

        if (ndopts.nd_opts_tgt_lladdr || ndopts.nd_opts_rh) {
                ND_PRINTK(2, warn, "RA: invalid RA options\n");
        }
out:
        /* Send a notify if RA changed managed/otherconf flags or
         * timer settings or ra_mtu value
         */
        if (send_ifinfo_notify)
                inet6_ifinfo_notify(RTM_NEWLINK, in6_dev);

        fib6_info_release(rt);
        if (neigh)
                neigh_release(neigh);
        return reason;
}

static enum skb_drop_reason ndisc_redirect_rcv(struct sk_buff *skb)
{
        struct rd_msg *msg = (struct rd_msg *)skb_transport_header(skb);
        u32 ndoptlen = skb_tail_pointer(skb) - (skb_transport_header(skb) +
                                    offsetof(struct rd_msg, opt));
        struct ndisc_options ndopts;
        SKB_DR(reason);
        u8 *hdr;

#ifdef CONFIG_IPV6_NDISC_NODETYPE
        switch (skb->ndisc_nodetype) {
        case NDISC_NODETYPE_HOST:
        case NDISC_NODETYPE_NODEFAULT:
                ND_PRINTK(2, warn,
                          "Redirect: from host or unauthorized router\n");
                return reason;
        }
#endif

        if (!(ipv6_addr_type(&ipv6_hdr(skb)->saddr) & IPV6_ADDR_LINKLOCAL)) {
                ND_PRINTK(2, warn,
                          "Redirect: source address is not link-local\n");
                return reason;
        }

        if (!ndisc_parse_options(skb->dev, msg->opt, ndoptlen, &ndopts))
                return SKB_DROP_REASON_IPV6_NDISC_BAD_OPTIONS;

        if (!ndopts.nd_opts_rh) {
                ip6_redirect_no_header(skb, dev_net(skb->dev),
                                        skb->dev->ifindex);
                return reason;
        }

        hdr = (u8 *)ndopts.nd_opts_rh;
        hdr += 8;
        if (!pskb_pull(skb, hdr - skb_transport_header(skb)))
                return SKB_DROP_REASON_PKT_TOO_SMALL;

        return icmpv6_notify(skb, NDISC_REDIRECT, 0, 0);
}

static void ndisc_fill_redirect_hdr_option(struct sk_buff *skb,
                                           struct sk_buff *orig_skb,
                                           int rd_len)
{
        u8 *opt = skb_put(skb, rd_len);

        memset(opt, 0, 8);
        *(opt++) = ND_OPT_REDIRECT_HDR;
        *(opt++) = (rd_len >> 3);
        opt += 6;

        skb_copy_bits(orig_skb, skb_network_offset(orig_skb), opt,
                      rd_len - 8);
}

void ndisc_send_redirect(struct sk_buff *skb, const struct in6_addr *target)
{
        struct net_device *dev = skb->dev;
        struct net *net = dev_net(dev);
        struct sock *sk = net->ipv6.ndisc_sk;
        int optlen = 0;
        struct inet_peer *peer;
        struct sk_buff *buff;
        struct rd_msg *msg;
        struct in6_addr saddr_buf;
        struct rt6_info *rt;
        struct dst_entry *dst;
        struct flowi6 fl6;
        int rd_len;
        u8 ha_buf[MAX_ADDR_LEN], *ha = NULL,
           ops_data_buf[NDISC_OPS_REDIRECT_DATA_SPACE], *ops_data = NULL;
        bool ret;

        if (netif_is_l3_master(skb->dev)) {
                dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif);
                if (!dev)
                        return;
        }

        if (ipv6_get_lladdr(dev, &saddr_buf, IFA_F_TENTATIVE)) {
                ND_PRINTK(2, warn, "Redirect: no link-local address on %s\n",
                          dev->name);
                return;
        }

        if (!ipv6_addr_equal(&ipv6_hdr(skb)->daddr, target) &&
            ipv6_addr_type(target) != (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) {
                ND_PRINTK(2, warn,
                          "Redirect: target address is not link-local unicast\n");
                return;
        }

        icmpv6_flow_init(sk, &fl6, NDISC_REDIRECT,
                         &saddr_buf, &ipv6_hdr(skb)->saddr, dev->ifindex);

        dst = ip6_route_output(net, NULL, &fl6);
        if (dst->error) {
                dst_release(dst);
                return;
        }
        dst = xfrm_lookup(net, dst, flowi6_to_flowi(&fl6), NULL, 0);
        if (IS_ERR(dst))
                return;

        rt = dst_rt6_info(dst);

        if (rt->rt6i_flags & RTF_GATEWAY) {
                ND_PRINTK(2, warn,
                          "Redirect: destination is not a neighbour\n");
                goto release;
        }
        peer = inet_getpeer_v6(net->ipv6.peers, &ipv6_hdr(skb)->saddr, 1);
        ret = inet_peer_xrlim_allow(peer, 1*HZ);
        if (peer)
                inet_putpeer(peer);
        if (!ret)
                goto release;

        if (dev->addr_len) {
                struct neighbour *neigh = dst_neigh_lookup(skb_dst(skb), target);
                if (!neigh) {
                        ND_PRINTK(2, warn,
                                  "Redirect: no neigh for target address\n");
                        goto release;
                }

                read_lock_bh(&neigh->lock);
                if (neigh->nud_state & NUD_VALID) {
                        memcpy(ha_buf, neigh->ha, dev->addr_len);
                        read_unlock_bh(&neigh->lock);
                        ha = ha_buf;
                        optlen += ndisc_redirect_opt_addr_space(dev, neigh,
                                                                ops_data_buf,
                                                                &ops_data);
                } else
                        read_unlock_bh(&neigh->lock);

                neigh_release(neigh);
        }

        rd_len = min_t(unsigned int,
                       IPV6_MIN_MTU - sizeof(struct ipv6hdr) - sizeof(*msg) - optlen,
                       skb->len + 8);
        rd_len &= ~0x7;
        optlen += rd_len;

        buff = ndisc_alloc_skb(dev, sizeof(*msg) + optlen);
        if (!buff)
                goto release;

        msg = skb_put(buff, sizeof(*msg));
        *msg = (struct rd_msg) {
                .icmph = {
                        .icmp6_type = NDISC_REDIRECT,
                },
                .target = *target,
                .dest = ipv6_hdr(skb)->daddr,
        };

        /*
         *        include target_address option
         */

        if (ha)
                ndisc_fill_redirect_addr_option(buff, ha, ops_data);

        /*
         *        build redirect option and copy skb over to the new packet.
         */

        if (rd_len)
                ndisc_fill_redirect_hdr_option(buff, skb, rd_len);

        skb_dst_set(buff, dst);
        ndisc_send_skb(buff, &ipv6_hdr(skb)->saddr, &saddr_buf);
        return;

release:
        dst_release(dst);
}

static void pndisc_redo(struct sk_buff *skb)
{
        enum skb_drop_reason reason = ndisc_recv_ns(skb);

        kfree_skb_reason(skb, reason);
}

static int ndisc_is_multicast(const void *pkey)
{
        return ipv6_addr_is_multicast((struct in6_addr *)pkey);
}

static bool ndisc_suppress_frag_ndisc(struct sk_buff *skb)
{
        struct inet6_dev *idev = __in6_dev_get(skb->dev);

        if (!idev)
                return true;
        if (IP6CB(skb)->flags & IP6SKB_FRAGMENTED &&
            READ_ONCE(idev->cnf.suppress_frag_ndisc)) {
                net_warn_ratelimited("Received fragmented ndisc packet. Carefully consider disabling suppress_frag_ndisc.\n");
                return true;
        }
        return false;
}

enum skb_drop_reason ndisc_rcv(struct sk_buff *skb)
{
        struct nd_msg *msg;
        SKB_DR(reason);

        if (ndisc_suppress_frag_ndisc(skb))
                return SKB_DROP_REASON_IPV6_NDISC_FRAG;

        if (skb_linearize(skb))
                return SKB_DROP_REASON_NOMEM;

        msg = (struct nd_msg *)skb_transport_header(skb);

        __skb_push(skb, skb->data - skb_transport_header(skb));

        if (ipv6_hdr(skb)->hop_limit != 255) {
                ND_PRINTK(2, warn, "NDISC: invalid hop-limit: %d\n",
                          ipv6_hdr(skb)->hop_limit);
                return SKB_DROP_REASON_IPV6_NDISC_HOP_LIMIT;
        }

        if (msg->icmph.icmp6_code != 0) {
                ND_PRINTK(2, warn, "NDISC: invalid ICMPv6 code: %d\n",
                          msg->icmph.icmp6_code);
                return SKB_DROP_REASON_IPV6_NDISC_BAD_CODE;
        }

        switch (msg->icmph.icmp6_type) {
        case NDISC_NEIGHBOUR_SOLICITATION:
                memset(NEIGH_CB(skb), 0, sizeof(struct neighbour_cb));
                reason = ndisc_recv_ns(skb);
                break;

        case NDISC_NEIGHBOUR_ADVERTISEMENT:
                reason = ndisc_recv_na(skb);
                break;

        case NDISC_ROUTER_SOLICITATION:
                reason = ndisc_recv_rs(skb);
                break;

        case NDISC_ROUTER_ADVERTISEMENT:
                reason = ndisc_router_discovery(skb);
                break;

        case NDISC_REDIRECT:
                reason = ndisc_redirect_rcv(skb);
                break;
        }

        return reason;
}

static int ndisc_netdev_event(struct notifier_block *this, unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct netdev_notifier_change_info *change_info;
        struct net *net = dev_net(dev);
        struct inet6_dev *idev;
        bool evict_nocarrier;

        switch (event) {
        case NETDEV_CHANGEADDR:
                neigh_changeaddr(&nd_tbl, dev);
                fib6_run_gc(0, net, false);
                fallthrough;
        case NETDEV_UP:
                idev = in6_dev_get(dev);
                if (!idev)
                        break;
                if (READ_ONCE(idev->cnf.ndisc_notify) ||
                    READ_ONCE(net->ipv6.devconf_all->ndisc_notify))
                        ndisc_send_unsol_na(dev);
                in6_dev_put(idev);
                break;
        case NETDEV_CHANGE:
                idev = in6_dev_get(dev);
                if (!idev)
                        evict_nocarrier = true;
                else {
                        evict_nocarrier = READ_ONCE(idev->cnf.ndisc_evict_nocarrier) &&
                                          READ_ONCE(net->ipv6.devconf_all->ndisc_evict_nocarrier);
                        in6_dev_put(idev);
                }

                change_info = ptr;
                if (change_info->flags_changed & IFF_NOARP)
                        neigh_changeaddr(&nd_tbl, dev);
                if (evict_nocarrier && !netif_carrier_ok(dev))
                        neigh_carrier_down(&nd_tbl, dev);
                break;
        case NETDEV_DOWN:
                neigh_ifdown(&nd_tbl, dev);
                fib6_run_gc(0, net, false);
                break;
        case NETDEV_NOTIFY_PEERS:
                ndisc_send_unsol_na(dev);
                break;
        default:
                break;
        }

        return NOTIFY_DONE;
}

static struct notifier_block ndisc_netdev_notifier = {
        .notifier_call = ndisc_netdev_event,
        .priority = ADDRCONF_NOTIFY_PRIORITY - 5,
};

#ifdef CONFIG_SYSCTL
static void ndisc_warn_deprecated_sysctl(const struct ctl_table *ctl,
                                         const char *func, const char *dev_name)
{
        static char warncomm[TASK_COMM_LEN];
        static int warned;
        if (strcmp(warncomm, current->comm) && warned < 5) {
                strscpy(warncomm, current->comm);
                pr_warn("process `%s' is using deprecated sysctl (%s) net.ipv6.neigh.%s.%s - use net.ipv6.neigh.%s.%s_ms instead\n",
                        warncomm, func,
                        dev_name, ctl->procname,
                        dev_name, ctl->procname);
                warned++;
        }
}

int ndisc_ifinfo_sysctl_change(const struct ctl_table *ctl, int write, void *buffer,
                size_t *lenp, loff_t *ppos)
{
        struct net_device *dev = ctl->extra1;
        struct inet6_dev *idev;
        int ret;

        if ((strcmp(ctl->procname, "retrans_time") == 0) ||
            (strcmp(ctl->procname, "base_reachable_time") == 0))
                ndisc_warn_deprecated_sysctl(ctl, "syscall", dev ? dev->name : "default");

        if (strcmp(ctl->procname, "retrans_time") == 0)
                ret = neigh_proc_dointvec(ctl, write, buffer, lenp, ppos);

        else if (strcmp(ctl->procname, "base_reachable_time") == 0)
                ret = neigh_proc_dointvec_jiffies(ctl, write,
                                                  buffer, lenp, ppos);

        else if ((strcmp(ctl->procname, "retrans_time_ms") == 0) ||
                 (strcmp(ctl->procname, "base_reachable_time_ms") == 0))
                ret = neigh_proc_dointvec_ms_jiffies(ctl, write,
                                                     buffer, lenp, ppos);
        else
                ret = -1;

        if (write && ret == 0 && dev && (idev = in6_dev_get(dev)) != NULL) {
                if (ctl->data == &NEIGH_VAR(idev->nd_parms, BASE_REACHABLE_TIME))
                        idev->nd_parms->reachable_time =
                                        neigh_rand_reach_time(NEIGH_VAR(idev->nd_parms, BASE_REACHABLE_TIME));
                WRITE_ONCE(idev->tstamp, jiffies);
                inet6_ifinfo_notify(RTM_NEWLINK, idev);
                in6_dev_put(idev);
        }
        return ret;
}


#endif

static int __net_init ndisc_net_init(struct net *net)
{
        struct ipv6_pinfo *np;
        struct sock *sk;
        int err;

        err = inet_ctl_sock_create(&sk, PF_INET6,
                                   SOCK_RAW, IPPROTO_ICMPV6, net);
        if (err < 0) {
                ND_PRINTK(0, err,
                          "NDISC: Failed to initialize the control socket (err %d)\n",
                          err);
                return err;
        }

        net->ipv6.ndisc_sk = sk;

        np = inet6_sk(sk);
        np->hop_limit = 255;
        /* Do not loopback ndisc messages */
        inet6_clear_bit(MC6_LOOP, sk);

        return 0;
}

static void __net_exit ndisc_net_exit(struct net *net)
{
        inet_ctl_sock_destroy(net->ipv6.ndisc_sk);
}

static struct pernet_operations ndisc_net_ops = {
        .init = ndisc_net_init,
        .exit = ndisc_net_exit,
};

int __init ndisc_init(void)
{
        int err;

        err = register_pernet_subsys(&ndisc_net_ops);
        if (err)
                return err;
        /*
         * Initialize the neighbour table
         */
        neigh_table_init(NEIGH_ND_TABLE, &nd_tbl);

#ifdef CONFIG_SYSCTL
        err = neigh_sysctl_register(NULL, &nd_tbl.parms,
                                    ndisc_ifinfo_sysctl_change);
        if (err)
                goto out_unregister_pernet;
out:
#endif
        return err;

#ifdef CONFIG_SYSCTL
out_unregister_pernet:
        unregister_pernet_subsys(&ndisc_net_ops);
        goto out;
#endif
}

int __init ndisc_late_init(void)
{
        return register_netdevice_notifier(&ndisc_netdev_notifier);
}

void ndisc_late_cleanup(void)
{
        unregister_netdevice_notifier(&ndisc_netdev_notifier);
}

void ndisc_cleanup(void)
{
#ifdef CONFIG_SYSCTL
        neigh_sysctl_unregister(&nd_tbl.parms);
#endif
        neigh_table_clear(NEIGH_ND_TABLE, &nd_tbl);
        unregister_pernet_subsys(&ndisc_net_ops);
}














































































































































































    2 




    2 





    1 

    3 
    1 































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        connector.c
 *
 * 2004+ Copyright (c) Evgeniy Polyakov <zbr@ioremap.net>
 * All rights reserved.
 */

#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/skbuff.h>
#include <net/netlink.h>
#include <linux/moduleparam.h>
#include <linux/connector.h>
#include <linux/slab.h>
#include <linux/mutex.h>
#include <linux/proc_fs.h>
#include <linux/spinlock.h>

#include <net/sock.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Evgeniy Polyakov <zbr@ioremap.net>");
MODULE_DESCRIPTION("Generic userspace <-> kernelspace connector.");
MODULE_ALIAS_NET_PF_PROTO(PF_NETLINK, NETLINK_CONNECTOR);

static struct cn_dev cdev;

static int cn_already_initialized;

/*
 * Sends mult (multiple) cn_msg at a time.
 *
 * msg->seq and msg->ack are used to determine message genealogy.
 * When someone sends message it puts there locally unique sequence
 * and random acknowledge numbers.  Sequence number may be copied into
 * nlmsghdr->nlmsg_seq too.
 *
 * Sequence number is incremented with each message to be sent.
 *
 * If we expect a reply to our message then the sequence number in
 * received message MUST be the same as in original message, and
 * acknowledge number MUST be the same + 1.
 *
 * If we receive a message and its sequence number is not equal to the
 * one we are expecting then it is a new message.
 *
 * If we receive a message and its sequence number is the same as one
 * we are expecting but it's acknowledgement number is not equal to
 * the acknowledgement number in the original message + 1, then it is
 * a new message.
 *
 * If msg->len != len, then additional cn_msg messages are expected following
 * the first msg.
 *
 * The message is sent to, the portid if given, the group if given, both if
 * both, or if both are zero then the group is looked up and sent there.
 */
int cn_netlink_send_mult(struct cn_msg *msg, u16 len, u32 portid, u32 __group,
                         gfp_t gfp_mask, netlink_filter_fn filter,
                         void *filter_data)
{
        struct cn_callback_entry *__cbq;
        unsigned int size;
        struct sk_buff *skb;
        struct nlmsghdr *nlh;
        struct cn_msg *data;
        struct cn_dev *dev = &cdev;
        u32 group = 0;
        int found = 0;

        if (portid || __group) {
                group = __group;
        } else {
                spin_lock_bh(&dev->cbdev->queue_lock);
                list_for_each_entry(__cbq, &dev->cbdev->queue_list,
                                    callback_entry) {
                        if (cn_cb_equal(&__cbq->id.id, &msg->id)) {
                                found = 1;
                                group = __cbq->group;
                                break;
                        }
                }
                spin_unlock_bh(&dev->cbdev->queue_lock);

                if (!found)
                        return -ENODEV;
        }

        if (!portid && !netlink_has_listeners(dev->nls, group))
                return -ESRCH;

        size = sizeof(*msg) + len;

        skb = nlmsg_new(size, gfp_mask);
        if (!skb)
                return -ENOMEM;

        nlh = nlmsg_put(skb, 0, msg->seq, NLMSG_DONE, size, 0);
        if (!nlh) {
                kfree_skb(skb);
                return -EMSGSIZE;
        }

        data = nlmsg_data(nlh);

        memcpy(data, msg, size);

        NETLINK_CB(skb).dst_group = group;

        if (group)
                return netlink_broadcast_filtered(dev->nls, skb, portid, group,
                                                  gfp_mask, filter,
                                                  (void *)filter_data);
        return netlink_unicast(dev->nls, skb, portid,
                        !gfpflags_allow_blocking(gfp_mask));
}
EXPORT_SYMBOL_GPL(cn_netlink_send_mult);

/* same as cn_netlink_send_mult except msg->len is used for len */
int cn_netlink_send(struct cn_msg *msg, u32 portid, u32 __group,
        gfp_t gfp_mask)
{
        return cn_netlink_send_mult(msg, msg->len, portid, __group, gfp_mask,
                                    NULL, NULL);
}
EXPORT_SYMBOL_GPL(cn_netlink_send);

/*
 * Callback helper - queues work and setup destructor for given data.
 */
static int cn_call_callback(struct sk_buff *skb)
{
        struct nlmsghdr *nlh;
        struct cn_callback_entry *i, *cbq = NULL;
        struct cn_dev *dev = &cdev;
        struct cn_msg *msg = nlmsg_data(nlmsg_hdr(skb));
        struct netlink_skb_parms *nsp = &NETLINK_CB(skb);
        int err = -ENODEV;

        /* verify msg->len is within skb */
        nlh = nlmsg_hdr(skb);
        if (nlh->nlmsg_len < NLMSG_HDRLEN + sizeof(struct cn_msg) + msg->len)
                return -EINVAL;

        spin_lock_bh(&dev->cbdev->queue_lock);
        list_for_each_entry(i, &dev->cbdev->queue_list, callback_entry) {
                if (cn_cb_equal(&i->id.id, &msg->id)) {
                        refcount_inc(&i->refcnt);
                        cbq = i;
                        break;
                }
        }
        spin_unlock_bh(&dev->cbdev->queue_lock);

        if (cbq != NULL) {
                cbq->callback(msg, nsp);
                kfree_skb(skb);
                cn_queue_release_callback(cbq);
                err = 0;
        }

        return err;
}

/*
 * Allow non-root access for NETLINK_CONNECTOR family having CN_IDX_PROC
 * multicast group.
 */
static int cn_bind(struct net *net, int group)
{
        unsigned long groups = (unsigned long) group;

        if (ns_capable(net->user_ns, CAP_NET_ADMIN))
                return 0;

        if (test_bit(CN_IDX_PROC - 1, &groups))
                return 0;

        return -EPERM;
}

static void cn_release(struct sock *sk, unsigned long *groups)
{
        if (groups && test_bit(CN_IDX_PROC - 1, groups)) {
                kfree(sk->sk_user_data);
                sk->sk_user_data = NULL;
        }
}

/*
 * Main netlink receiving function.
 *
 * It checks skb, netlink header and msg sizes, and calls callback helper.
 */
static void cn_rx_skb(struct sk_buff *skb)
{
        struct nlmsghdr *nlh;
        int len, err;

        if (skb->len >= NLMSG_HDRLEN) {
                nlh = nlmsg_hdr(skb);
                len = nlmsg_len(nlh);

                if (len < (int)sizeof(struct cn_msg) ||
                    skb->len < nlh->nlmsg_len ||
                    len > CONNECTOR_MAX_MSG_SIZE)
                        return;

                err = cn_call_callback(skb_get(skb));
                if (err < 0)
                        kfree_skb(skb);
        }
}

/*
 * Callback add routing - adds callback with given ID and name.
 * If there is registered callback with the same ID it will not be added.
 *
 * May sleep.
 */
int cn_add_callback(const struct cb_id *id, const char *name,
                    void (*callback)(struct cn_msg *,
                                     struct netlink_skb_parms *))
{
        struct cn_dev *dev = &cdev;

        if (!cn_already_initialized)
                return -EAGAIN;

        return cn_queue_add_callback(dev->cbdev, name, id, callback);
}
EXPORT_SYMBOL_GPL(cn_add_callback);

/*
 * Callback remove routing - removes callback
 * with given ID.
 * If there is no registered callback with given
 * ID nothing happens.
 *
 * May sleep while waiting for reference counter to become zero.
 */
void cn_del_callback(const struct cb_id *id)
{
        struct cn_dev *dev = &cdev;

        cn_queue_del_callback(dev->cbdev, id);
}
EXPORT_SYMBOL_GPL(cn_del_callback);

static int __maybe_unused cn_proc_show(struct seq_file *m, void *v)
{
        struct cn_queue_dev *dev = cdev.cbdev;
        struct cn_callback_entry *cbq;

        seq_printf(m, "Name            ID\n");

        spin_lock_bh(&dev->queue_lock);

        list_for_each_entry(cbq, &dev->queue_list, callback_entry) {
                seq_printf(m, "%-15s %u:%u\n",
                           cbq->id.name,
                           cbq->id.id.idx,
                           cbq->id.id.val);
        }

        spin_unlock_bh(&dev->queue_lock);

        return 0;
}

static int cn_init(void)
{
        struct cn_dev *dev = &cdev;
        struct netlink_kernel_cfg cfg = {
                .groups        = CN_NETLINK_USERS + 0xf,
                .input        = cn_rx_skb,
                .flags  = NL_CFG_F_NONROOT_RECV,
                .bind   = cn_bind,
                .release = cn_release,
        };

        dev->nls = netlink_kernel_create(&init_net, NETLINK_CONNECTOR, &cfg);
        if (!dev->nls)
                return -EIO;

        dev->cbdev = cn_queue_alloc_dev("cqueue", dev->nls);
        if (!dev->cbdev) {
                netlink_kernel_release(dev->nls);
                return -EINVAL;
        }

        cn_already_initialized = 1;

        proc_create_single("connector", S_IRUGO, init_net.proc_net, cn_proc_show);

        return 0;
}

static void cn_fini(void)
{
        struct cn_dev *dev = &cdev;

        cn_already_initialized = 0;

        remove_proc_entry("connector", init_net.proc_net);

        cn_queue_free_dev(dev->cbdev);
        netlink_kernel_release(dev->nls);
}

subsys_initcall(cn_init);
module_exit(cn_fini);




























    4 




    1 





















    8 




    1 
































































    8 




    8 

































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_UIDGID_H
#define _LINUX_UIDGID_H

/*
 * A set of types for the internal kernel types representing uids and gids.
 *
 * The types defined in this header allow distinguishing which uids and gids in
 * the kernel are values used by userspace and which uid and gid values are
 * the internal kernel values.  With the addition of user namespaces the values
 * can be different.  Using the type system makes it possible for the compiler
 * to detect when we overlook these differences.
 *
 */
#include <linux/uidgid_types.h>
#include <linux/highuid.h>

struct user_namespace;
extern struct user_namespace init_user_ns;
struct uid_gid_map;

#define KUIDT_INIT(value) (kuid_t){ value }
#define KGIDT_INIT(value) (kgid_t){ value }

#ifdef CONFIG_MULTIUSER
static inline uid_t __kuid_val(kuid_t uid)
{
        return uid.val;
}

static inline gid_t __kgid_val(kgid_t gid)
{
        return gid.val;
}
#else
static inline uid_t __kuid_val(kuid_t uid)
{
        return 0;
}

static inline gid_t __kgid_val(kgid_t gid)
{
        return 0;
}
#endif

#define GLOBAL_ROOT_UID KUIDT_INIT(0)
#define GLOBAL_ROOT_GID KGIDT_INIT(0)

#define INVALID_UID KUIDT_INIT(-1)
#define INVALID_GID KGIDT_INIT(-1)

static inline bool uid_eq(kuid_t left, kuid_t right)
{
        return __kuid_val(left) == __kuid_val(right);
}

static inline bool gid_eq(kgid_t left, kgid_t right)
{
        return __kgid_val(left) == __kgid_val(right);
}

static inline bool uid_gt(kuid_t left, kuid_t right)
{
        return __kuid_val(left) > __kuid_val(right);
}

static inline bool gid_gt(kgid_t left, kgid_t right)
{
        return __kgid_val(left) > __kgid_val(right);
}

static inline bool uid_gte(kuid_t left, kuid_t right)
{
        return __kuid_val(left) >= __kuid_val(right);
}

static inline bool gid_gte(kgid_t left, kgid_t right)
{
        return __kgid_val(left) >= __kgid_val(right);
}

static inline bool uid_lt(kuid_t left, kuid_t right)
{
        return __kuid_val(left) < __kuid_val(right);
}

static inline bool gid_lt(kgid_t left, kgid_t right)
{
        return __kgid_val(left) < __kgid_val(right);
}

static inline bool uid_lte(kuid_t left, kuid_t right)
{
        return __kuid_val(left) <= __kuid_val(right);
}

static inline bool gid_lte(kgid_t left, kgid_t right)
{
        return __kgid_val(left) <= __kgid_val(right);
}

static inline bool uid_valid(kuid_t uid)
{
        return __kuid_val(uid) != (uid_t) -1;
}

static inline bool gid_valid(kgid_t gid)
{
        return __kgid_val(gid) != (gid_t) -1;
}

#ifdef CONFIG_USER_NS

extern kuid_t make_kuid(struct user_namespace *from, uid_t uid);
extern kgid_t make_kgid(struct user_namespace *from, gid_t gid);

extern uid_t from_kuid(struct user_namespace *to, kuid_t uid);
extern gid_t from_kgid(struct user_namespace *to, kgid_t gid);
extern uid_t from_kuid_munged(struct user_namespace *to, kuid_t uid);
extern gid_t from_kgid_munged(struct user_namespace *to, kgid_t gid);

static inline bool kuid_has_mapping(struct user_namespace *ns, kuid_t uid)
{
        return from_kuid(ns, uid) != (uid_t) -1;
}

static inline bool kgid_has_mapping(struct user_namespace *ns, kgid_t gid)
{
        return from_kgid(ns, gid) != (gid_t) -1;
}

u32 map_id_down(struct uid_gid_map *map, u32 id);
u32 map_id_up(struct uid_gid_map *map, u32 id);

#else

static inline kuid_t make_kuid(struct user_namespace *from, uid_t uid)
{
        return KUIDT_INIT(uid);
}

static inline kgid_t make_kgid(struct user_namespace *from, gid_t gid)
{
        return KGIDT_INIT(gid);
}

static inline uid_t from_kuid(struct user_namespace *to, kuid_t kuid)
{
        return __kuid_val(kuid);
}

static inline gid_t from_kgid(struct user_namespace *to, kgid_t kgid)
{
        return __kgid_val(kgid);
}

static inline uid_t from_kuid_munged(struct user_namespace *to, kuid_t kuid)
{
        uid_t uid = from_kuid(to, kuid);
        if (uid == (uid_t)-1)
                uid = overflowuid;
        return uid;
}

static inline gid_t from_kgid_munged(struct user_namespace *to, kgid_t kgid)
{
        gid_t gid = from_kgid(to, kgid);
        if (gid == (gid_t)-1)
                gid = overflowgid;
        return gid;
}

static inline bool kuid_has_mapping(struct user_namespace *ns, kuid_t uid)
{
        return uid_valid(uid);
}

static inline bool kgid_has_mapping(struct user_namespace *ns, kgid_t gid)
{
        return gid_valid(gid);
}

static inline u32 map_id_down(struct uid_gid_map *map, u32 id)
{
        return id;
}

static inline u32 map_id_up(struct uid_gid_map *map, u32 id)
{
        return id;
}
#endif /* CONFIG_USER_NS */

#endif /* _LINUX_UIDGID_H */










































































































































































































































































































































































































































































  261 


  260 


    1 

























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        LAPB release 002
 *
 *        This code REQUIRES 2.1.15 or higher/ NET3.038
 *
 *        History
 *        LAPB 001        Jonathan Naylor        Started Coding
 *        LAPB 002