Total coverage: 164773 (9%)of 1952228 










































































































   10 


























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
// SPDX-License-Identifier: GPL-2.0
/*
 * The class-specific portions of the driver model
 *
 * Copyright (c) 2001-2003 Patrick Mochel <mochel@osdl.org>
 * Copyright (c) 2004-2009 Greg Kroah-Hartman <gregkh@suse.de>
 * Copyright (c) 2008-2009 Novell Inc.
 * Copyright (c) 2012-2019 Greg Kroah-Hartman <gregkh@linuxfoundation.org>
 * Copyright (c) 2012-2019 Linux Foundation
 *
 * See Documentation/driver-api/driver-model/ for more information.
 */

#ifndef _DEVICE_CLASS_H_
#define _DEVICE_CLASS_H_

#include <linux/kobject.h>
#include <linux/klist.h>
#include <linux/pm.h>
#include <linux/device/bus.h>

struct device;
struct fwnode_handle;

/**
 * struct class - device classes
 * @name:        Name of the class.
 * @class_groups: Default attributes of this class.
 * @dev_groups:        Default attributes of the devices that belong to the class.
 * @dev_uevent:        Called when a device is added, removed from this class, or a
 *                few other things that generate uevents to add the environment
 *                variables.
 * @devnode:        Callback to provide the devtmpfs.
 * @class_release: Called to release this class.
 * @dev_release: Called to release the device.
 * @shutdown_pre: Called at shut-down time before driver shutdown.
 * @ns_type:        Callbacks so sysfs can determine namespaces.
 * @namespace:        Namespace of the device belongs to this class.
 * @get_ownership: Allows class to specify uid/gid of the sysfs directories
 *                for the devices belonging to the class. Usually tied to
 *                device's namespace.
 * @pm:                The default device power management operations of this class.
 *
 * A class is a higher-level view of a device that abstracts out low-level
 * implementation details. Drivers may see a SCSI disk or an ATA disk, but,
 * at the class level, they are all simply disks. Classes allow user space
 * to work with devices based on what they do, rather than how they are
 * connected or how they work.
 */
struct class {
        const char                *name;

        const struct attribute_group        *const *class_groups;
        const struct attribute_group        *const *dev_groups;

        int (*dev_uevent)(const struct device *dev, struct kobj_uevent_env *env);
        char *(*devnode)(const struct device *dev, umode_t *mode);

        void (*class_release)(const struct class *class);
        void (*dev_release)(struct device *dev);

        int (*shutdown_pre)(struct device *dev);

        const struct kobj_ns_type_operations *ns_type;
        const struct ns_common *(*namespace)(const struct device *dev);

        void (*get_ownership)(const struct device *dev, kuid_t *uid, kgid_t *gid);

        const struct dev_pm_ops *pm;
};

struct class_dev_iter {
        struct klist_iter                ki;
        const struct device_type        *type;
        struct subsys_private                *sp;
};

int __must_check class_register(const struct class *class);
void class_unregister(const struct class *class);
bool class_is_registered(const struct class *class);

struct class_compat;
struct class_compat *class_compat_register(const char *name);
void class_compat_unregister(struct class_compat *cls);
int class_compat_create_link(struct class_compat *cls, struct device *dev);
void class_compat_remove_link(struct class_compat *cls, struct device *dev);

void class_dev_iter_init(struct class_dev_iter *iter, const struct class *class,
                         const struct device *start, const struct device_type *type);
struct device *class_dev_iter_next(struct class_dev_iter *iter);
void class_dev_iter_exit(struct class_dev_iter *iter);

int class_for_each_device(const struct class *class, const struct device *start,
                          void *data, device_iter_t fn);
struct device *class_find_device(const struct class *class, const struct device *start,
                                 const void *data, device_match_t match);

/**
 * class_find_device_by_name - device iterator for locating a particular device
 * of a specific name.
 * @class: class type
 * @name: name of the device to match
 */
static inline struct device *class_find_device_by_name(const struct class *class,
                                                       const char *name)
{
        return class_find_device(class, NULL, name, device_match_name);
}

/**
 * class_find_device_by_of_node : device iterator for locating a particular device
 * matching the of_node.
 * @class: class type
 * @np: of_node of the device to match.
 */
static inline struct device *class_find_device_by_of_node(const struct class *class,
                                                          const struct device_node *np)
{
        return class_find_device(class, NULL, np, device_match_of_node);
}

/**
 * class_find_device_by_fwnode : device iterator for locating a particular device
 * matching the fwnode.
 * @class: class type
 * @fwnode: fwnode of the device to match.
 */
static inline struct device *class_find_device_by_fwnode(const struct class *class,
                                                         const struct fwnode_handle *fwnode)
{
        return class_find_device(class, NULL, fwnode, device_match_fwnode);
}

/**
 * class_find_device_by_devt : device iterator for locating a particular device
 * matching the device type.
 * @class: class type
 * @devt: device type of the device to match.
 */
static inline struct device *class_find_device_by_devt(const struct class *class,
                                                       dev_t devt)
{
        return class_find_device(class, NULL, &devt, device_match_devt);
}

#ifdef CONFIG_ACPI
struct acpi_device;
/**
 * class_find_device_by_acpi_dev : device iterator for locating a particular
 * device matching the ACPI_COMPANION device.
 * @class: class type
 * @adev: ACPI_COMPANION device to match.
 */
static inline struct device *class_find_device_by_acpi_dev(const struct class *class,
                                                           const struct acpi_device *adev)
{
        return class_find_device(class, NULL, adev, device_match_acpi_dev);
}
#else
static inline struct device *class_find_device_by_acpi_dev(const struct class *class,
                                                           const void *adev)
{
        return NULL;
}
#endif

struct class_attribute {
        struct attribute attr;
        ssize_t (*show)(const struct class *class, const struct class_attribute *attr,
                        char *buf);
        ssize_t (*store)(const struct class *class, const struct class_attribute *attr,
                         const char *buf, size_t count);
};

#define CLASS_ATTR_RW(_name) \
        struct class_attribute class_attr_##_name = __ATTR_RW(_name)
#define CLASS_ATTR_RO(_name) \
        struct class_attribute class_attr_##_name = __ATTR_RO(_name)
#define CLASS_ATTR_WO(_name) \
        struct class_attribute class_attr_##_name = __ATTR_WO(_name)

int __must_check class_create_file_ns(const struct class *class, const struct class_attribute *attr,
                                      const struct ns_common *ns);
void class_remove_file_ns(const struct class *class, const struct class_attribute *attr,
                          const struct ns_common *ns);

static inline int __must_check class_create_file(const struct class *class,
                                                 const struct class_attribute *attr)
{
        return class_create_file_ns(class, attr, NULL);
}

static inline void class_remove_file(const struct class *class,
                                     const struct class_attribute *attr)
{
        class_remove_file_ns(class, attr, NULL);
}

/* Simple class attribute that is just a static string */
struct class_attribute_string {
        struct class_attribute attr;
        char *str;
};

/* Currently read-only only */
#define _CLASS_ATTR_STRING(_name, _mode, _str) \
        { __ATTR(_name, _mode, show_class_attr_string, NULL), _str }
#define CLASS_ATTR_STRING(_name, _mode, _str) \
        struct class_attribute_string class_attr_##_name = \
                _CLASS_ATTR_STRING(_name, _mode, _str)

ssize_t show_class_attr_string(const struct class *class, const struct class_attribute *attr,
                               char *buf);

struct class_interface {
        struct list_head        node;
        const struct class        *class;

        int (*add_dev)                (struct device *dev);
        void (*remove_dev)        (struct device *dev);
};

int __must_check class_interface_register(struct class_interface *);
void class_interface_unregister(struct class_interface *);

struct class * __must_check class_create(const char *name);
void class_destroy(const struct class *cls);

#endif        /* _DEVICE_CLASS_H_ */
























































   25 


















 4297 



  566 











 3880 






















 1109 
  681 



  560 

























  820 
  820 



    4 























 2402 
 2402 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
// SPDX-License-Identifier: GPL-2.0
/*
 * Out-of-line refcount functions.
 */

#include <linux/mutex.h>
#include <linux/refcount.h>
#include <linux/spinlock.h>
#include <linux/bug.h>

#define REFCOUNT_WARN(str)        WARN_ONCE(1, "refcount_t: " str ".\n")

void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t)
{
        refcount_set(r, REFCOUNT_SATURATED);

        switch (t) {
        case REFCOUNT_ADD_NOT_ZERO_OVF:
                REFCOUNT_WARN("saturated; leaking memory");
                break;
        case REFCOUNT_ADD_OVF:
                REFCOUNT_WARN("saturated; leaking memory");
                break;
        case REFCOUNT_ADD_UAF:
                REFCOUNT_WARN("addition on 0; use-after-free");
                break;
        case REFCOUNT_SUB_UAF:
                REFCOUNT_WARN("underflow; use-after-free");
                break;
        case REFCOUNT_DEC_LEAK:
                REFCOUNT_WARN("decrement hit 0; leaking memory");
                break;
        default:
                REFCOUNT_WARN("unknown saturation event!?");
        }
}
EXPORT_SYMBOL(refcount_warn_saturate);

/**
 * refcount_dec_if_one - decrement a refcount if it is 1
 * @r: the refcount
 *
 * No atomic_t counterpart, it attempts a 1 -> 0 transition and returns the
 * success thereof.
 *
 * Like all decrement operations, it provides release memory order and provides
 * a control dependency.
 *
 * It can be used like a try-delete operator; this explicit case is provided
 * and not cmpxchg in generic, because that would allow implementing unsafe
 * operations.
 *
 * Return: true if the resulting refcount is 0, false otherwise
 */
bool refcount_dec_if_one(refcount_t *r)
{
        int val = 1;

        return atomic_try_cmpxchg_release(&r->refs, &val, 0);
}
EXPORT_SYMBOL(refcount_dec_if_one);

/**
 * refcount_dec_not_one - decrement a refcount if it is not 1
 * @r: the refcount
 *
 * No atomic_t counterpart, it decrements unless the value is 1, in which case
 * it will return false.
 *
 * Was often done like: atomic_add_unless(&var, -1, 1)
 *
 * Return: true if the decrement operation was successful, false otherwise
 */
bool refcount_dec_not_one(refcount_t *r)
{
        unsigned int new, val = atomic_read(&r->refs);

        do {
                if (unlikely(val == REFCOUNT_SATURATED))
                        return true;

                if (val == 1)
                        return false;

                new = val - 1;
                if (new > val) {
                        WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
                        return true;
                }

        } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));

        return true;
}
EXPORT_SYMBOL(refcount_dec_not_one);

/**
 * refcount_dec_and_mutex_lock - return holding mutex if able to decrement
 *                               refcount to 0
 * @r: the refcount
 * @lock: the mutex to be locked
 *
 * Similar to atomic_dec_and_mutex_lock(), it will WARN on underflow and fail
 * to decrement when saturated at REFCOUNT_SATURATED.
 *
 * Provides release memory ordering, such that prior loads and stores are done
 * before, and provides a control dependency such that free() must come after.
 * See the comment on top.
 *
 * Return: true and hold mutex if able to decrement refcount to 0, false
 *         otherwise
 */
bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock)
{
        if (refcount_dec_not_one(r))
                return false;

        mutex_lock(lock);
        if (!refcount_dec_and_test(r)) {
                mutex_unlock(lock);
                return false;
        }

        return true;
}
EXPORT_SYMBOL(refcount_dec_and_mutex_lock);

/**
 * refcount_dec_and_lock - return holding spinlock if able to decrement
 *                         refcount to 0
 * @r: the refcount
 * @lock: the spinlock to be locked
 *
 * Similar to atomic_dec_and_lock(), it will WARN on underflow and fail to
 * decrement when saturated at REFCOUNT_SATURATED.
 *
 * Provides release memory ordering, such that prior loads and stores are done
 * before, and provides a control dependency such that free() must come after.
 * See the comment on top.
 *
 * Return: true and hold spinlock if able to decrement refcount to 0, false
 *         otherwise
 */
bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock)
{
        if (refcount_dec_not_one(r))
                return false;

        spin_lock(lock);
        if (!refcount_dec_and_test(r)) {
                spin_unlock(lock);
                return false;
        }

        return true;
}
EXPORT_SYMBOL(refcount_dec_and_lock);

/**
 * refcount_dec_and_lock_irqsave - return holding spinlock with disabled
 *                                 interrupts if able to decrement refcount to 0
 * @r: the refcount
 * @lock: the spinlock to be locked
 * @flags: saved IRQ-flags if the is acquired
 *
 * Same as refcount_dec_and_lock() above except that the spinlock is acquired
 * with disabled interrupts.
 *
 * Return: true and hold spinlock if able to decrement refcount to 0, false
 *         otherwise
 */
bool refcount_dec_and_lock_irqsave(refcount_t *r, spinlock_t *lock,
                                   unsigned long *flags)
{
        if (refcount_dec_not_one(r))
                return false;

        spin_lock_irqsave(lock, *flags);
        if (!refcount_dec_and_test(r)) {
                spin_unlock_irqrestore(lock, *flags);
                return false;
        }

        return true;
}
EXPORT_SYMBOL(refcount_dec_and_lock_irqsave);



























































































































































































































   29 















   29 




















   29 













   29 






   29 
















   29 














   29 


   29 











   29 


   29 












   29 

































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * RSA Signature Scheme with Appendix - PKCS #1 v1.5 (RFC 8017 sec 8.2)
 *
 * https://www.rfc-editor.org/rfc/rfc8017#section-8.2
 *
 * Copyright (c) 2015 - 2024 Intel Corporation
 */

#include <linux/module.h>
#include <linux/scatterlist.h>
#include <crypto/akcipher.h>
#include <crypto/algapi.h>
#include <crypto/hash.h>
#include <crypto/sig.h>
#include <crypto/internal/akcipher.h>
#include <crypto/internal/rsa.h>
#include <crypto/internal/sig.h>

/*
 * Full Hash Prefix for EMSA-PKCS1-v1_5 encoding method (RFC 9580 table 24)
 *
 * RSA keys are usually much larger than the hash of the message to be signed.
 * The hash is therefore prepended by the Full Hash Prefix and a 0xff padding.
 * The Full Hash Prefix is an ASN.1 SEQUENCE containing the hash algorithm OID.
 *
 * https://www.rfc-editor.org/rfc/rfc9580#table-24
 */

static const u8 hash_prefix_none[] = { };

static const u8 hash_prefix_md5[] = {
        0x30, 0x20, 0x30, 0x0c, 0x06, 0x08,          /* SEQUENCE (SEQUENCE (OID */
        0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05,        /*        <algorithm>, */
        0x05, 0x00, 0x04, 0x10                      /* NULL), OCTET STRING <hash>) */
};

static const u8 hash_prefix_sha1[] = {
        0x30, 0x21, 0x30, 0x09, 0x06, 0x05,
        0x2b, 0x0e, 0x03, 0x02, 0x1a,
        0x05, 0x00, 0x04, 0x14
};

static const u8 hash_prefix_rmd160[] = {
        0x30, 0x21, 0x30, 0x09, 0x06, 0x05,
        0x2b, 0x24, 0x03, 0x02, 0x01,
        0x05, 0x00, 0x04, 0x14
};

static const u8 hash_prefix_sha224[] = {
        0x30, 0x2d, 0x30, 0x0d, 0x06, 0x09,
        0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x04,
        0x05, 0x00, 0x04, 0x1c
};

static const u8 hash_prefix_sha256[] = {
        0x30, 0x31, 0x30, 0x0d, 0x06, 0x09,
        0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01,
        0x05, 0x00, 0x04, 0x20
};

static const u8 hash_prefix_sha384[] = {
        0x30, 0x41, 0x30, 0x0d, 0x06, 0x09,
        0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02,
        0x05, 0x00, 0x04, 0x30
};

static const u8 hash_prefix_sha512[] = {
        0x30, 0x51, 0x30, 0x0d, 0x06, 0x09,
        0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03,
        0x05, 0x00, 0x04, 0x40
};

static const u8 hash_prefix_sha3_256[] = {
        0x30, 0x31, 0x30, 0x0d, 0x06, 0x09,
        0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x08,
        0x05, 0x00, 0x04, 0x20
};

static const u8 hash_prefix_sha3_384[] = {
        0x30, 0x41, 0x30, 0x0d, 0x06, 0x09,
        0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x09,
        0x05, 0x00, 0x04, 0x30
};

static const u8 hash_prefix_sha3_512[] = {
        0x30, 0x51, 0x30, 0x0d, 0x06, 0x09,
        0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x0a,
        0x05, 0x00, 0x04, 0x40
};

static const struct hash_prefix {
        const char        *name;
        const u8        *data;
        size_t                size;
} hash_prefixes[] = {
#define _(X) { #X, hash_prefix_##X, sizeof(hash_prefix_##X) }
        _(none),
        _(md5),
        _(sha1),
        _(rmd160),
        _(sha256),
        _(sha384),
        _(sha512),
        _(sha224),
#undef _
#define _(X) { "sha3-" #X, hash_prefix_sha3_##X, sizeof(hash_prefix_sha3_##X) }
        _(256),
        _(384),
        _(512),
#undef _
        { NULL }
};

static const struct hash_prefix *rsassa_pkcs1_find_hash_prefix(const char *name)
{
        const struct hash_prefix *p;

        for (p = hash_prefixes; p->name; p++)
                if (strcmp(name, p->name) == 0)
                        return p;
        return NULL;
}

static bool rsassa_pkcs1_invalid_hash_len(unsigned int len,
                                          const struct hash_prefix *p)
{
        /*
         * Legacy protocols such as TLS 1.1 or earlier and IKE version 1
         * do not prepend a Full Hash Prefix to the hash.  In that case,
         * the size of the Full Hash Prefix is zero.
         */
        if (p->data == hash_prefix_none)
                return false;

        /*
         * The final byte of the Full Hash Prefix encodes the hash length.
         *
         * This needs to be revisited should hash algorithms with more than
         * 1016 bits (127 bytes * 8) ever be added.  The length would then
         * be encoded into more than one byte by ASN.1.
         */
        static_assert(HASH_MAX_DIGESTSIZE <= 127);

        return len != p->data[p->size - 1];
}

struct rsassa_pkcs1_ctx {
        struct crypto_akcipher *child;
        unsigned int key_size;
};

struct rsassa_pkcs1_inst_ctx {
        struct crypto_akcipher_spawn spawn;
        const struct hash_prefix *hash_prefix;
};

static int rsassa_pkcs1_sign(struct crypto_sig *tfm,
                             const void *src, unsigned int slen,
                             void *dst, unsigned int dlen)
{
        struct sig_instance *inst = sig_alg_instance(tfm);
        struct rsassa_pkcs1_inst_ctx *ictx = sig_instance_ctx(inst);
        const struct hash_prefix *hash_prefix = ictx->hash_prefix;
        struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm);
        unsigned int pad_len;
        unsigned int ps_end;
        unsigned int len;
        u8 *in_buf;
        int err;

        if (!ctx->key_size)
                return -EINVAL;

        if (dlen < ctx->key_size)
                return -EOVERFLOW;

        if (rsassa_pkcs1_invalid_hash_len(slen, hash_prefix))
                return -EINVAL;

        if (slen + hash_prefix->size > ctx->key_size - 11)
                return -EOVERFLOW;

        pad_len = ctx->key_size - slen - hash_prefix->size - 1;

        /* RFC 8017 sec 8.2.1 step 1 - EMSA-PKCS1-v1_5 encoding generation */
        in_buf = dst;
        memmove(in_buf + pad_len + hash_prefix->size, src, slen);
        memcpy(in_buf + pad_len, hash_prefix->data, hash_prefix->size);

        ps_end = pad_len - 1;
        in_buf[0] = 0x01;
        memset(in_buf + 1, 0xff, ps_end - 1);
        in_buf[ps_end] = 0x00;


        /* RFC 8017 sec 8.2.1 step 2 - RSA signature */
        err = crypto_akcipher_sync_decrypt(ctx->child, in_buf,
                                           ctx->key_size - 1, in_buf,
                                           ctx->key_size);
        if (err < 0)
                return err;

        len = err;
        pad_len = ctx->key_size - len;

        /* Four billion to one */
        if (unlikely(pad_len)) {
                memmove(dst + pad_len, dst, len);
                memset(dst, 0, pad_len);
        }

        return ctx->key_size;
}

static int rsassa_pkcs1_verify(struct crypto_sig *tfm,
                               const void *src, unsigned int slen,
                               const void *digest, unsigned int dlen)
{
        struct sig_instance *inst = sig_alg_instance(tfm);
        struct rsassa_pkcs1_inst_ctx *ictx = sig_instance_ctx(inst);
        const struct hash_prefix *hash_prefix = ictx->hash_prefix;
        struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm);
        unsigned int child_reqsize = crypto_akcipher_reqsize(ctx->child);
        struct akcipher_request *child_req __free(kfree_sensitive) = NULL;
        struct crypto_wait cwait;
        struct scatterlist sg;
        unsigned int dst_len;
        unsigned int pos;
        u8 *out_buf;
        int err;

        /* RFC 8017 sec 8.2.2 step 1 - length checking */
        if (!ctx->key_size ||
            slen != ctx->key_size ||
            rsassa_pkcs1_invalid_hash_len(dlen, hash_prefix))
                return -EINVAL;

        /* RFC 8017 sec 8.2.2 step 2 - RSA verification */
        child_req = kmalloc(sizeof(*child_req) + child_reqsize + ctx->key_size,
                            GFP_KERNEL);
        if (!child_req)
                return -ENOMEM;

        out_buf = (u8 *)(child_req + 1) + child_reqsize;
        memcpy(out_buf, src, slen);

        crypto_init_wait(&cwait);
        sg_init_one(&sg, out_buf, slen);
        akcipher_request_set_tfm(child_req, ctx->child);
        akcipher_request_set_crypt(child_req, &sg, &sg, slen, slen);
        akcipher_request_set_callback(child_req, CRYPTO_TFM_REQ_MAY_SLEEP,
                                      crypto_req_done, &cwait);

        err = crypto_akcipher_encrypt(child_req);
        err = crypto_wait_req(err, &cwait);
        if (err)
                return err;

        /* RFC 8017 sec 8.2.2 step 3 - EMSA-PKCS1-v1_5 encoding verification */
        dst_len = child_req->dst_len;
        if (dst_len < ctx->key_size - 1)
                return -EINVAL;

        if (dst_len == ctx->key_size) {
                if (out_buf[0] != 0x00)
                        /* Encrypted value had no leading 0 byte */
                        return -EINVAL;

                dst_len--;
                out_buf++;
        }

        if (out_buf[0] != 0x01)
                return -EBADMSG;

        for (pos = 1; pos < dst_len; pos++)
                if (out_buf[pos] != 0xff)
                        break;

        if (pos < 9 || pos == dst_len || out_buf[pos] != 0x00)
                return -EBADMSG;
        pos++;

        if (hash_prefix->size > dst_len - pos)
                return -EBADMSG;
        if (crypto_memneq(out_buf + pos, hash_prefix->data, hash_prefix->size))
                return -EBADMSG;
        pos += hash_prefix->size;

        /* RFC 8017 sec 8.2.2 step 4 - comparison of digest with out_buf */
        if (dlen != dst_len - pos)
                return -EKEYREJECTED;
        if (memcmp(digest, out_buf + pos, dlen) != 0)
                return -EKEYREJECTED;

        return 0;
}

static unsigned int rsassa_pkcs1_key_size(struct crypto_sig *tfm)
{
        struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm);

        return ctx->key_size * BITS_PER_BYTE;
}

static int rsassa_pkcs1_set_pub_key(struct crypto_sig *tfm,
                                    const void *key, unsigned int keylen)
{
        struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm);

        return rsa_set_key(ctx->child, &ctx->key_size, RSA_PUB, key, keylen);
}

static int rsassa_pkcs1_set_priv_key(struct crypto_sig *tfm,
                                     const void *key, unsigned int keylen)
{
        struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm);

        return rsa_set_key(ctx->child, &ctx->key_size, RSA_PRIV, key, keylen);
}

static int rsassa_pkcs1_init_tfm(struct crypto_sig *tfm)
{
        struct sig_instance *inst = sig_alg_instance(tfm);
        struct rsassa_pkcs1_inst_ctx *ictx = sig_instance_ctx(inst);
        struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm);
        struct crypto_akcipher *child_tfm;

        child_tfm = crypto_spawn_akcipher(&ictx->spawn);
        if (IS_ERR(child_tfm))
                return PTR_ERR(child_tfm);

        ctx->child = child_tfm;

        return 0;
}

static void rsassa_pkcs1_exit_tfm(struct crypto_sig *tfm)
{
        struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm);

        crypto_free_akcipher(ctx->child);
}

static void rsassa_pkcs1_free(struct sig_instance *inst)
{
        struct rsassa_pkcs1_inst_ctx *ctx = sig_instance_ctx(inst);
        struct crypto_akcipher_spawn *spawn = &ctx->spawn;

        crypto_drop_akcipher(spawn);
        kfree(inst);
}

static int rsassa_pkcs1_create(struct crypto_template *tmpl, struct rtattr **tb)
{
        struct rsassa_pkcs1_inst_ctx *ctx;
        struct akcipher_alg *rsa_alg;
        struct sig_instance *inst;
        const char *hash_name;
        u32 mask;
        int err;

        err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SIG, &mask);
        if (err)
                return err;

        inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
        if (!inst)
                return -ENOMEM;

        ctx = sig_instance_ctx(inst);

        err = crypto_grab_akcipher(&ctx->spawn, sig_crypto_instance(inst),
                                   crypto_attr_alg_name(tb[1]), 0, mask);
        if (err)
                goto err_free_inst;

        rsa_alg = crypto_spawn_akcipher_alg(&ctx->spawn);

        if (strcmp(rsa_alg->base.cra_name, "rsa") != 0) {
                err = -EINVAL;
                goto err_free_inst;
        }

        hash_name = crypto_attr_alg_name(tb[2]);
        if (IS_ERR(hash_name)) {
                err = PTR_ERR(hash_name);
                goto err_free_inst;
        }

        ctx->hash_prefix = rsassa_pkcs1_find_hash_prefix(hash_name);
        if (!ctx->hash_prefix) {
                err = -EINVAL;
                goto err_free_inst;
        }

        err = -ENAMETOOLONG;
        if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
                     "pkcs1(%s,%s)", rsa_alg->base.cra_name,
                     hash_name) >= CRYPTO_MAX_ALG_NAME)
                goto err_free_inst;

        if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
                     "pkcs1(%s,%s)", rsa_alg->base.cra_driver_name,
                     hash_name) >= CRYPTO_MAX_ALG_NAME)
                goto err_free_inst;

        inst->alg.base.cra_priority = rsa_alg->base.cra_priority;
        inst->alg.base.cra_ctxsize = sizeof(struct rsassa_pkcs1_ctx);

        inst->alg.init = rsassa_pkcs1_init_tfm;
        inst->alg.exit = rsassa_pkcs1_exit_tfm;

        inst->alg.sign = rsassa_pkcs1_sign;
        inst->alg.verify = rsassa_pkcs1_verify;
        inst->alg.key_size = rsassa_pkcs1_key_size;
        inst->alg.set_pub_key = rsassa_pkcs1_set_pub_key;
        inst->alg.set_priv_key = rsassa_pkcs1_set_priv_key;

        inst->free = rsassa_pkcs1_free;

        err = sig_register_instance(tmpl, inst);
        if (err) {
err_free_inst:
                rsassa_pkcs1_free(inst);
        }
        return err;
}

struct crypto_template rsassa_pkcs1_tmpl = {
        .name = "pkcs1",
        .create = rsassa_pkcs1_create,
        .module = THIS_MODULE,
};

MODULE_ALIAS_CRYPTO("pkcs1");


















































































    2 


    1 

    1 






































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
// SPDX-License-Identifier: GPL-2.0-only
/* Kernel module to match TCP MSS values. */

/* Copyright (C) 2000 Marc Boucher <marc@mbsi.ca>
 * Portions (C) 2005 by Harald Welte <laforge@netfilter.org>
 */

#include <linux/module.h>
#include <linux/skbuff.h>
#include <net/tcp.h>

#include <linux/netfilter/xt_tcpmss.h>
#include <linux/netfilter/x_tables.h>

#include <linux/netfilter_ipv4/ip_tables.h>
#include <linux/netfilter_ipv6/ip6_tables.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Marc Boucher <marc@mbsi.ca>");
MODULE_DESCRIPTION("Xtables: TCP MSS match");
MODULE_ALIAS("ipt_tcpmss");
MODULE_ALIAS("ip6t_tcpmss");

static bool
tcpmss_mt(const struct sk_buff *skb, struct xt_action_param *par)
{
        const struct xt_tcpmss_match_info *info = par->matchinfo;
        const struct tcphdr *th;
        struct tcphdr _tcph;
        /* tcp.doff is only 4 bits, ie. max 15 * 4 bytes */
        const u_int8_t *op;
        u8 _opt[15 * 4 - sizeof(_tcph)];
        unsigned int i, optlen;

        /* this is fine for IPv6 as xt_tcpmss enforces -p tcp */
        if (par->fragoff)
                return false;

        /* If we don't have the whole header, drop packet. */
        th = skb_header_pointer(skb, par->thoff, sizeof(_tcph), &_tcph);
        if (th == NULL)
                goto dropit;

        /* Malformed. */
        if (th->doff*4 < sizeof(*th))
                goto dropit;

        optlen = th->doff*4 - sizeof(*th);
        if (!optlen)
                goto out;

        /* Truncated options. */
        op = skb_header_pointer(skb, par->thoff + sizeof(*th), optlen, _opt);
        if (op == NULL)
                goto dropit;

        for (i = 0; i < optlen; ) {
                if (op[i] == TCPOPT_MSS
                    && (optlen - i) >= TCPOLEN_MSS
                    && op[i+1] == TCPOLEN_MSS) {
                        u_int16_t mssval;

                        mssval = (op[i+2] << 8) | op[i+3];

                        return (mssval >= info->mss_min &&
                                mssval <= info->mss_max) ^ info->invert;
                }
                if (op[i] < 2 || i == optlen - 1)
                        i++;
                else
                        i += op[i+1] ? : 1;
        }
out:
        return info->invert;

dropit:
        par->hotdrop = true;
        return false;
}

static int tcpmss_mt_check(const struct xt_mtchk_param *par)
{
        const struct xt_tcpmss_match_info *info = par->matchinfo;

        if (info->mss_min > info->mss_max)
                return -EINVAL;
        if (info->invert > 1)
                return -EINVAL;

        return 0;
}

static struct xt_match tcpmss_mt_reg[] __read_mostly = {
        {
                .name                = "tcpmss",
                .family                = NFPROTO_IPV4,
                .checkentry        = tcpmss_mt_check,
                .match                = tcpmss_mt,
                .matchsize        = sizeof(struct xt_tcpmss_match_info),
                .proto                = IPPROTO_TCP,
                .me                = THIS_MODULE,
        },
        {
                .name                = "tcpmss",
                .family                = NFPROTO_IPV6,
                .checkentry        = tcpmss_mt_check,
                .match                = tcpmss_mt,
                .matchsize        = sizeof(struct xt_tcpmss_match_info),
                .proto                = IPPROTO_TCP,
                .me                = THIS_MODULE,
        },
};

static int __init tcpmss_mt_init(void)
{
        return xt_register_matches(tcpmss_mt_reg, ARRAY_SIZE(tcpmss_mt_reg));
}

static void __exit tcpmss_mt_exit(void)
{
        xt_unregister_matches(tcpmss_mt_reg, ARRAY_SIZE(tcpmss_mt_reg));
}

module_init(tcpmss_mt_init);
module_exit(tcpmss_mt_exit);



































































































    3 










    1 


    2 










    9 


    2 









    4 


    2 










    3 


    3 









   12 


    3 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (C) B.A.T.M.A.N. contributors:
 *
 * Marek Lindner, Simon Wunderlich
 */

#ifndef _NET_BATMAN_ADV_ORIGINATOR_H_
#define _NET_BATMAN_ADV_ORIGINATOR_H_

#include "main.h"

#include <linux/compiler.h>
#include <linux/if_ether.h>
#include <linux/jhash.h>
#include <linux/kref.h>
#include <linux/netlink.h>
#include <linux/skbuff.h>
#include <linux/types.h>

bool batadv_compare_orig(const struct hlist_node *node, const void *data2);
int batadv_originator_init(struct batadv_priv *bat_priv);
void batadv_originator_free(struct batadv_priv *bat_priv);
void batadv_purge_orig_ref(struct batadv_priv *bat_priv);
void batadv_orig_node_release(struct kref *ref);
struct batadv_orig_node *batadv_orig_node_new(struct batadv_priv *bat_priv,
                                              const u8 *addr);
struct batadv_hardif_neigh_node *
batadv_hardif_neigh_get(const struct batadv_hard_iface *hard_iface,
                        const u8 *neigh_addr);
void batadv_hardif_neigh_release(struct kref *ref);
struct batadv_neigh_node *
batadv_neigh_node_get_or_create(struct batadv_orig_node *orig_node,
                                struct batadv_hard_iface *hard_iface,
                                const u8 *neigh_addr);
void batadv_neigh_node_release(struct kref *ref);
struct batadv_neigh_node *
batadv_orig_router_get(struct batadv_orig_node *orig_node,
                       const struct batadv_hard_iface *if_outgoing);
struct batadv_neigh_node *
batadv_orig_to_router(struct batadv_priv *bat_priv, u8 *orig_addr,
                      struct batadv_hard_iface *if_outgoing);
struct batadv_neigh_ifinfo *
batadv_neigh_ifinfo_new(struct batadv_neigh_node *neigh,
                        struct batadv_hard_iface *if_outgoing);
struct batadv_neigh_ifinfo *
batadv_neigh_ifinfo_get(struct batadv_neigh_node *neigh,
                        struct batadv_hard_iface *if_outgoing);
void batadv_neigh_ifinfo_release(struct kref *ref);

int batadv_hardif_neigh_dump(struct sk_buff *msg, struct netlink_callback *cb);

struct batadv_orig_ifinfo *
batadv_orig_ifinfo_get(struct batadv_orig_node *orig_node,
                       struct batadv_hard_iface *if_outgoing);
struct batadv_orig_ifinfo *
batadv_orig_ifinfo_new(struct batadv_orig_node *orig_node,
                       struct batadv_hard_iface *if_outgoing);
void batadv_orig_ifinfo_release(struct kref *ref);

int batadv_orig_dump(struct sk_buff *msg, struct netlink_callback *cb);
struct batadv_orig_node_vlan *
batadv_orig_node_vlan_new(struct batadv_orig_node *orig_node,
                          unsigned short vid);
struct batadv_orig_node_vlan *
batadv_orig_node_vlan_get(struct batadv_orig_node *orig_node,
                          unsigned short vid);
void batadv_orig_node_vlan_release(struct kref *ref);

/**
 * batadv_choose_orig() - Return the index of the orig entry in the hash table
 * @data: mac address of the originator node
 * @size: the size of the hash table
 *
 * Return: the hash index where the object represented by @data should be
 * stored at.
 */
static inline u32 batadv_choose_orig(const void *data, u32 size)
{
        u32 hash = 0;

        hash = jhash(data, ETH_ALEN, hash);
        return hash % size;
}

struct batadv_orig_node *
batadv_orig_hash_find(struct batadv_priv *bat_priv, const void *data);

/**
 * batadv_orig_node_vlan_put() - decrement the refcounter and possibly release
 *  the originator-vlan object
 * @orig_vlan: the originator-vlan object to release
 */
static inline void
batadv_orig_node_vlan_put(struct batadv_orig_node_vlan *orig_vlan)
{
        if (!orig_vlan)
                return;

        kref_put(&orig_vlan->refcount, batadv_orig_node_vlan_release);
}

/**
 * batadv_neigh_ifinfo_put() - decrement the refcounter and possibly release
 *  the neigh_ifinfo
 * @neigh_ifinfo: the neigh_ifinfo object to release
 */
static inline void
batadv_neigh_ifinfo_put(struct batadv_neigh_ifinfo *neigh_ifinfo)
{
        if (!neigh_ifinfo)
                return;

        kref_put(&neigh_ifinfo->refcount, batadv_neigh_ifinfo_release);
}

/**
 * batadv_hardif_neigh_put() - decrement the hardif neighbors refcounter
 *  and possibly release it
 * @hardif_neigh: hardif neigh neighbor to free
 */
static inline void
batadv_hardif_neigh_put(struct batadv_hardif_neigh_node *hardif_neigh)
{
        if (!hardif_neigh)
                return;

        kref_put(&hardif_neigh->refcount, batadv_hardif_neigh_release);
}

/**
 * batadv_neigh_node_put() - decrement the neighbors refcounter and possibly
 *  release it
 * @neigh_node: neigh neighbor to free
 */
static inline void batadv_neigh_node_put(struct batadv_neigh_node *neigh_node)
{
        if (!neigh_node)
                return;

        kref_put(&neigh_node->refcount, batadv_neigh_node_release);
}

/**
 * batadv_orig_ifinfo_put() - decrement the refcounter and possibly release
 *  the orig_ifinfo
 * @orig_ifinfo: the orig_ifinfo object to release
 */
static inline void
batadv_orig_ifinfo_put(struct batadv_orig_ifinfo *orig_ifinfo)
{
        if (!orig_ifinfo)
                return;

        kref_put(&orig_ifinfo->refcount, batadv_orig_ifinfo_release);
}

/**
 * batadv_orig_node_put() - decrement the orig node refcounter and possibly
 *  release it
 * @orig_node: the orig node to free
 */
static inline void batadv_orig_node_put(struct batadv_orig_node *orig_node)
{
        if (!orig_node)
                return;

        kref_put(&orig_node->refcount, batadv_orig_node_release);
}

#endif /* _NET_BATMAN_ADV_ORIGINATOR_H_ */
















    1 





    8 

    1 










    9 






















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
 */

#ifndef RXE_PARAM_H
#define RXE_PARAM_H

#include <uapi/rdma/rdma_user_rxe.h>

#define DEFAULT_MAX_VALUE (1 << 20)

static inline enum ib_mtu rxe_mtu_int_to_enum(int mtu)
{
        if (mtu < 256)
                return 0;
        else if (mtu < 512)
                return IB_MTU_256;
        else if (mtu < 1024)
                return IB_MTU_512;
        else if (mtu < 2048)
                return IB_MTU_1024;
        else if (mtu < 4096)
                return IB_MTU_2048;
        else
                return IB_MTU_4096;
}

/* Find the IB mtu for a given network MTU. */
static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)
{
        mtu -= RXE_MAX_HDR_LENGTH;

        return rxe_mtu_int_to_enum(mtu);
}

/* default/initial rxe device parameter settings */
enum rxe_device_param {
        RXE_MAX_MR_SIZE                        = -1ull,
        RXE_PAGE_SIZE_CAP                = 0xfffff000,
        RXE_MAX_QP_WR                        = DEFAULT_MAX_VALUE,
        RXE_DEVICE_CAP_FLAGS                = IB_DEVICE_BAD_PKEY_CNTR
                                        | IB_DEVICE_BAD_QKEY_CNTR
                                        | IB_DEVICE_AUTO_PATH_MIG
                                        | IB_DEVICE_CHANGE_PHY_PORT
                                        | IB_DEVICE_UD_AV_PORT_ENFORCE
                                        | IB_DEVICE_PORT_ACTIVE_EVENT
                                        | IB_DEVICE_SYS_IMAGE_GUID
                                        | IB_DEVICE_RC_RNR_NAK_GEN
                                        | IB_DEVICE_SRQ_RESIZE
                                        | IB_DEVICE_MEM_MGT_EXTENSIONS
                                        | IB_DEVICE_MEM_WINDOW
                                        | IB_DEVICE_FLUSH_GLOBAL
                                        | IB_DEVICE_FLUSH_PERSISTENT
                                        | IB_DEVICE_MEM_WINDOW_TYPE_2B
                                        | IB_DEVICE_ATOMIC_WRITE,

        RXE_MAX_SGE                        = 32,
        RXE_MAX_WQE_SIZE                = sizeof(struct rxe_send_wqe) +
                                          sizeof(struct ib_sge) * RXE_MAX_SGE,
        RXE_MAX_INLINE_DATA                = RXE_MAX_WQE_SIZE -
                                          sizeof(struct rxe_send_wqe),
        RXE_MAX_SGE_RD                        = 32,
        RXE_MAX_CQ                        = DEFAULT_MAX_VALUE,
        RXE_MAX_LOG_CQE                        = 15,
        RXE_MAX_PD                        = DEFAULT_MAX_VALUE,
        RXE_MAX_QP_RD_ATOM                = 128,
        RXE_MAX_RES_RD_ATOM                = 0x3f000,
        RXE_MAX_QP_INIT_RD_ATOM                = 128,
        RXE_MAX_MCAST_GRP                = 8192,
        RXE_MAX_MCAST_QP_ATTACH                = 56,
        RXE_MAX_TOT_MCAST_QP_ATTACH        = 0x70000,
        RXE_MAX_AH                        = (1<<15) - 1,        /* 32Ki - 1 */
        RXE_MIN_AH_INDEX                = 1,
        RXE_MAX_AH_INDEX                = RXE_MAX_AH,
        RXE_MAX_SRQ_WR                        = DEFAULT_MAX_VALUE,
        RXE_MIN_SRQ_WR                        = 1,
        RXE_MAX_SRQ_SGE                        = 27,
        RXE_MIN_SRQ_SGE                        = 1,
        RXE_MAX_FMR_PAGE_LIST_LEN        = 512,
        RXE_MAX_PKEYS                        = 64,
        RXE_LOCAL_CA_ACK_DELAY                = 15,

        RXE_MAX_UCONTEXT                = DEFAULT_MAX_VALUE,

        RXE_NUM_PORT                        = 1,

        RXE_MIN_QP_INDEX                = 16,
        RXE_MAX_QP_INDEX                = DEFAULT_MAX_VALUE,
        RXE_MAX_QP                        = DEFAULT_MAX_VALUE - RXE_MIN_QP_INDEX,

        RXE_MIN_SRQ_INDEX                = 0x00020001,
        RXE_MAX_SRQ_INDEX                = DEFAULT_MAX_VALUE,
        RXE_MAX_SRQ                        = DEFAULT_MAX_VALUE - RXE_MIN_SRQ_INDEX,

        RXE_MIN_MR_INDEX                = 0x00000001,
        RXE_MAX_MR_INDEX                = DEFAULT_MAX_VALUE >> 1,
        RXE_MAX_MR                        = RXE_MAX_MR_INDEX - RXE_MIN_MR_INDEX,
        RXE_MIN_MW_INDEX                = RXE_MAX_MR_INDEX + 1,
        RXE_MAX_MW_INDEX                = DEFAULT_MAX_VALUE,
        RXE_MAX_MW                        = RXE_MAX_MW_INDEX - RXE_MIN_MW_INDEX,

        RXE_MAX_PKT_PER_ACK                = 64,

        RXE_MAX_UNACKED_PSNS                = 128,

        /* Max inflight SKBs per queue pair */
        RXE_INFLIGHT_SKBS_PER_QP_HIGH        = 64,
        RXE_INFLIGHT_SKBS_PER_QP_LOW        = 16,

        /* Max number of iterations of each work item
         * before yielding the cpu to let other
         * work make progress
         */
        RXE_MAX_ITERATIONS                = 1024,

        /* Delay before calling arbiter timer */
        RXE_NSEC_ARB_TIMER_DELAY        = 200,

        /* IBTA v1.4 A3.3.1 VENDOR INFORMATION section */
        RXE_VENDOR_ID                        = 0XFFFFFF,
};

/* default/initial rxe port parameters */
enum rxe_port_param {
        RXE_PORT_GID_TBL_LEN                = 1024,
        RXE_PORT_PORT_CAP_FLAGS                = IB_PORT_CM_SUP,
        RXE_PORT_MAX_MSG_SZ                = (1UL << 31),
        RXE_PORT_BAD_PKEY_CNTR                = 0,
        RXE_PORT_QKEY_VIOL_CNTR                = 0,
        RXE_PORT_LID                        = 0,
        RXE_PORT_SM_LID                        = 0,
        RXE_PORT_SM_SL                        = 0,
        RXE_PORT_LMC                        = 0,
        RXE_PORT_MAX_VL_NUM                = 1,
        RXE_PORT_SUBNET_TIMEOUT                = 0,
        RXE_PORT_INIT_TYPE_REPLY        = 0,
        RXE_PORT_ACTIVE_WIDTH                = IB_WIDTH_1X,
        RXE_PORT_ACTIVE_SPEED                = 1,
        RXE_PORT_PKEY_TBL_LEN                = 1,
        RXE_PORT_PHYS_STATE                = IB_PORT_PHYS_STATE_POLLING,
        RXE_PORT_SUBNET_PREFIX                = 0xfe80000000000000ULL,
};

/* default/initial port info parameters */
enum rxe_port_info_param {
        RXE_PORT_INFO_VL_CAP                = 4,        /* 1-8 */
        RXE_PORT_INFO_MTU_CAP                = 5,        /* 4096 */
        RXE_PORT_INFO_OPER_VL                = 1,        /* 1 */
};

#endif /* RXE_PARAM_H */





















































































































































































































































































































































 18132 






 18132 




































































  468 







  468 






































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
// SPDX-License-Identifier: GPL-2.0-only
/* Common code for 32 and 64-bit NUMA */
#include <linux/acpi.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/of.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/memblock.h>
#include <linux/mmzone.h>
#include <linux/ctype.h>
#include <linux/nodemask.h>
#include <linux/sched.h>
#include <linux/topology.h>
#include <linux/sort.h>
#include <linux/numa_memblks.h>

#include <asm/e820/api.h>
#include <asm/proto.h>
#include <asm/dma.h>
#include <asm/numa.h>
#include <asm/amd/nb.h>

#include "mm_internal.h"

int numa_off;

static __init int numa_setup(char *opt)
{
        if (!opt)
                return -EINVAL;
        if (!strncmp(opt, "off", 3))
                numa_off = 1;
        if (!strncmp(opt, "fake=", 5))
                return numa_emu_cmdline(opt + 5);
        if (!strncmp(opt, "noacpi", 6))
                disable_srat();
        if (!strncmp(opt, "nohmat", 6))
                disable_hmat();
        return 0;
}
early_param("numa", numa_setup);

/*
 * apicid, cpu, node mappings
 */
s16 __apicid_to_node[MAX_LOCAL_APIC] = {
        [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
};

nodemask_t numa_phys_nodes_parsed __initdata;

int numa_cpu_node(int cpu)
{
        u32 apicid = early_per_cpu(x86_cpu_to_apicid, cpu);

        if (apicid != BAD_APICID)
                return __apicid_to_node[apicid];
        return NUMA_NO_NODE;
}

int __init num_phys_nodes(void)
{
        return bitmap_weight(numa_phys_nodes_parsed.bits, MAX_NUMNODES);
}

cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
EXPORT_SYMBOL(node_to_cpumask_map);

/*
 * Map cpu index to node index
 */
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);

void numa_set_node(int cpu, int node)
{
        int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);

        /* early setting, no percpu area yet */
        if (cpu_to_node_map) {
                cpu_to_node_map[cpu] = node;
                return;
        }

#ifdef CONFIG_DEBUG_PER_CPU_MAPS
        if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
                printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
                dump_stack();
                return;
        }
#endif
        per_cpu(x86_cpu_to_node_map, cpu) = node;

        set_cpu_numa_node(cpu, node);
}

void numa_clear_node(int cpu)
{
        numa_set_node(cpu, NUMA_NO_NODE);
}

/*
 * Allocate node_to_cpumask_map based on number of available nodes
 * Requires node_possible_map to be valid.
 *
 * Note: cpumask_of_node() is not valid until after this is done.
 * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
 */
void __init setup_node_to_cpumask_map(void)
{
        unsigned int node;

        /* setup nr_node_ids if not done yet */
        if (nr_node_ids == MAX_NUMNODES)
                setup_nr_node_ids();

        /* allocate the map */
        for (node = 0; node < nr_node_ids; node++)
                alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);

        /* cpumask_of_node() will now work */
        pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids);
}

static int __init numa_register_nodes(void)
{
        int nid;

        if (!memblock_validate_numa_coverage(SZ_1M))
                return -EINVAL;

        /* Finally register nodes. */
        for_each_node_mask(nid, node_possible_map) {
                unsigned long start_pfn, end_pfn;

                /*
                 * Note, get_pfn_range_for_nid() depends on
                 * memblock_set_node() having already happened
                 */
                get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
                if (start_pfn >= end_pfn)
                        continue;

                alloc_node_data(nid);
                node_set_online(nid);
        }

        /* Dump memblock with node info and return. */
        memblock_dump_all();
        return 0;
}

/*
 * There are unfortunately some poorly designed mainboards around that
 * only connect memory to a single CPU. This breaks the 1:1 cpu->node
 * mapping. To avoid this fill in the mapping for all possible CPUs,
 * as the number of CPUs is not known yet. We round robin the existing
 * nodes.
 */
static void __init numa_init_array(void)
{
        int rr, i;

        rr = first_node(node_online_map);
        for (i = 0; i < nr_cpu_ids; i++) {
                if (early_cpu_to_node(i) != NUMA_NO_NODE)
                        continue;
                numa_set_node(i, rr);
                rr = next_node_in(rr, node_online_map);
        }
}

static int __init numa_init(int (*init_func)(void))
{
        int i;
        int ret;

        for (i = 0; i < MAX_LOCAL_APIC; i++)
                set_apicid_to_node(i, NUMA_NO_NODE);

        ret = numa_memblks_init(init_func, /* memblock_force_top_down */ true);
        if (ret < 0)
                return ret;

        ret = numa_register_nodes();
        if (ret < 0)
                return ret;

        for (i = 0; i < nr_cpu_ids; i++) {
                int nid = early_cpu_to_node(i);

                if (nid == NUMA_NO_NODE)
                        continue;
                if (!node_online(nid))
                        numa_clear_node(i);
        }
        numa_init_array();

        return 0;
}

/**
 * dummy_numa_init - Fallback dummy NUMA init
 *
 * Used if there's no underlying NUMA architecture, NUMA initialization
 * fails, or NUMA is disabled on the command line.
 *
 * Must online at least one node and add memory blocks that cover all
 * allowed memory.  This function must not fail.
 */
static int __init dummy_numa_init(void)
{
        printk(KERN_INFO "%s\n",
               numa_off ? "NUMA turned off" : "No NUMA configuration found");
        printk(KERN_INFO "Faking a node at [mem %#018Lx-%#018Lx]\n",
               0LLU, PFN_PHYS(max_pfn) - 1);

        node_set(0, numa_phys_nodes_parsed);
        numa_add_memblk(0, 0, PFN_PHYS(max_pfn));

        return 0;
}

/**
 * x86_numa_init - Initialize NUMA
 *
 * Try each configured NUMA initialization method until one succeeds.  The
 * last fallback is dummy single node config encompassing whole memory and
 * never fails.
 */
void __init x86_numa_init(void)
{
        if (!numa_off) {
#ifdef CONFIG_ACPI_NUMA
                if (!numa_init(x86_acpi_numa_init))
                        return;
#endif
#ifdef CONFIG_AMD_NUMA
                if (!numa_init(amd_numa_init))
                        return;
#endif
                if (acpi_disabled && !numa_init(of_numa_init))
                        return;
        }

        numa_init(dummy_numa_init);
}


/*
 * A node may exist which has one or more Generic Initiators but no CPUs and no
 * memory.
 *
 * This function must be called after init_cpu_to_node(), to ensure that any
 * memoryless CPU nodes have already been brought online, and before the
 * node_data[nid] is needed for zone list setup in build_all_zonelists().
 *
 * When this function is called, any nodes containing either memory and/or CPUs
 * will already be online and there is no need to do anything extra, even if
 * they also contain one or more Generic Initiators.
 */
void __init init_gi_nodes(void)
{
        int nid;

        /*
         * Exclude this node from
         * bringup_nonboot_cpus
         *  cpu_up
         *   __try_online_node
         *    register_node
         * because node_subsys is not initialized yet.
         * TODO remove dependency on node_online
         */
        for_each_node_state(nid, N_GENERIC_INITIATOR)
                if (!node_online(nid))
                        node_set_online(nid);
}

/*
 * Setup early cpu_to_node.
 *
 * Populate cpu_to_node[] only if x86_cpu_to_apicid[],
 * and apicid_to_node[] tables have valid entries for a CPU.
 * This means we skip cpu_to_node[] initialisation for NUMA
 * emulation and faking node case (when running a kernel compiled
 * for NUMA on a non NUMA box), which is OK as cpu_to_node[]
 * is already initialized in a round robin manner at numa_init_array,
 * prior to this call, and this initialization is good enough
 * for the fake NUMA cases.
 *
 * Called before the per_cpu areas are setup.
 */
void __init init_cpu_to_node(void)
{
        int cpu;
        u32 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);

        BUG_ON(cpu_to_apicid == NULL);

        for_each_possible_cpu(cpu) {
                int node = numa_cpu_node(cpu);

                if (node == NUMA_NO_NODE)
                        continue;

                /*
                 * Exclude this node from
                 * bringup_nonboot_cpus
                 *  cpu_up
                 *   __try_online_node
                 *    register_node
                 * because node_subsys is not initialized yet.
                 * TODO remove dependency on node_online
                 */
                if (!node_online(node))
                        node_set_online(node);

                numa_set_node(cpu, node);
        }
}

#ifndef CONFIG_DEBUG_PER_CPU_MAPS

# ifndef CONFIG_NUMA_EMU
void numa_add_cpu(unsigned int cpu)
{
        cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
}

void numa_remove_cpu(unsigned int cpu)
{
        cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
}
# endif        /* !CONFIG_NUMA_EMU */

#else        /* !CONFIG_DEBUG_PER_CPU_MAPS */

int __cpu_to_node(int cpu)
{
        if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
                printk(KERN_WARNING
                        "cpu_to_node(%d): usage too early!\n", cpu);
                dump_stack();
                return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
        }
        return per_cpu(x86_cpu_to_node_map, cpu);
}
EXPORT_SYMBOL(__cpu_to_node);

/*
 * Same function as cpu_to_node() but used if called before the
 * per_cpu areas are setup.
 */
int early_cpu_to_node(int cpu)
{
        if (early_per_cpu_ptr(x86_cpu_to_node_map))
                return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];

        if (!cpu_possible(cpu)) {
                printk(KERN_WARNING
                        "early_cpu_to_node(%d): no per_cpu area!\n", cpu);
                dump_stack();
                return NUMA_NO_NODE;
        }
        return per_cpu(x86_cpu_to_node_map, cpu);
}

void debug_cpumask_set_cpu(unsigned int cpu, int node, bool enable)
{
        struct cpumask *mask;

        if (node == NUMA_NO_NODE) {
                /* early_cpu_to_node() already emits a warning and trace */
                return;
        }
        mask = node_to_cpumask_map[node];
        if (!cpumask_available(mask)) {
                pr_err("node_to_cpumask_map[%i] NULL\n", node);
                dump_stack();
                return;
        }

        if (enable)
                cpumask_set_cpu(cpu, mask);
        else
                cpumask_clear_cpu(cpu, mask);

        printk(KERN_DEBUG "%s cpu %d node %d: mask now %*pbl\n",
                enable ? "numa_add_cpu" : "numa_remove_cpu",
                cpu, node, cpumask_pr_args(mask));
        return;
}

# ifndef CONFIG_NUMA_EMU
static void numa_set_cpumask(int cpu, bool enable)
{
        debug_cpumask_set_cpu(cpu, early_cpu_to_node(cpu), enable);
}

void numa_add_cpu(unsigned int cpu)
{
        numa_set_cpumask(cpu, true);
}

void numa_remove_cpu(unsigned int cpu)
{
        numa_set_cpumask(cpu, false);
}
# endif        /* !CONFIG_NUMA_EMU */

/*
 * Returns a pointer to the bitmask of CPUs on Node 'node'.
 */
const struct cpumask *cpumask_of_node(int node)
{
        if ((unsigned)node >= nr_node_ids) {
                printk(KERN_WARNING
                        "cpumask_of_node(%d): (unsigned)node >= nr_node_ids(%u)\n",
                        node, nr_node_ids);
                dump_stack();
                return cpu_none_mask;
        }
        if (!cpumask_available(node_to_cpumask_map[node])) {
                printk(KERN_WARNING
                        "cpumask_of_node(%d): no node_to_cpumask_map!\n",
                        node);
                dump_stack();
                return cpu_online_mask;
        }
        return node_to_cpumask_map[node];
}
EXPORT_SYMBOL(cpumask_of_node);

#endif        /* !CONFIG_DEBUG_PER_CPU_MAPS */

#ifdef CONFIG_NUMA_EMU
void __init numa_emu_update_cpu_to_node(int *emu_nid_to_phys,
                                        unsigned int nr_emu_nids)
{
        int i, j;

        /*
         * Transform __apicid_to_node table to use emulated nids by
         * reverse-mapping phys_nid.  The maps should always exist but fall
         * back to zero just in case.
         */
        for (i = 0; i < ARRAY_SIZE(__apicid_to_node); i++) {
                if (__apicid_to_node[i] == NUMA_NO_NODE)
                        continue;
                for (j = 0; j < nr_emu_nids; j++)
                        if (__apicid_to_node[i] == emu_nid_to_phys[j])
                                break;
                __apicid_to_node[i] = j < nr_emu_nids ? j : 0;
        }
}

u64 __init numa_emu_dma_end(void)
{
        return PFN_PHYS(MAX_DMA32_PFN);
}
#endif /* CONFIG_NUMA_EMU */



























    1 







    1 





















   44 













    1 




    1 


    5 








   30 

    2 


    1 



    3 
    1 










   32 
    1 



    1 


















   28 
    2 










   30 




    2 






    4 







    2 
    2 


    4 
    3 
    1 

    4 






    2 











   12 














   11 










   12 


   12 










   31 














   32 



   14 

    1 

    2 

    1 


    2 
    2 

    1 
    1 

    4 
    2 


    2 






   12 

    2 

    1 

    6 























   25 


   25 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/act_gact.c                Generic actions
 *
 * copyright         Jamal Hadi Salim (2002-4)
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/module.h>
#include <linux/init.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <linux/tc_act/tc_gact.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_wrapper.h>

static struct tc_action_ops act_gact_ops;

#ifdef CONFIG_GACT_PROB
static int gact_net_rand(struct tcf_gact *gact)
{
        smp_rmb(); /* coupled with smp_wmb() in tcf_gact_init() */
        if (get_random_u32_below(gact->tcfg_pval))
                return gact->tcf_action;
        return gact->tcfg_paction;
}

static int gact_determ(struct tcf_gact *gact)
{
        u32 pack = atomic_inc_return(&gact->packets);

        smp_rmb(); /* coupled with smp_wmb() in tcf_gact_init() */
        if (pack % gact->tcfg_pval)
                return gact->tcf_action;
        return gact->tcfg_paction;
}

typedef int (*g_rand)(struct tcf_gact *gact);
static g_rand gact_rand[MAX_RAND] = { NULL, gact_net_rand, gact_determ };
#endif /* CONFIG_GACT_PROB */

static const struct nla_policy gact_policy[TCA_GACT_MAX + 1] = {
        [TCA_GACT_PARMS]        = { .len = sizeof(struct tc_gact) },
        [TCA_GACT_PROB]                = { .len = sizeof(struct tc_gact_p) },
};

static int tcf_gact_init(struct net *net, struct nlattr *nla,
                         struct nlattr *est, struct tc_action **a,
                         struct tcf_proto *tp, u32 flags,
                         struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, act_gact_ops.net_id);
        bool bind = flags & TCA_ACT_FLAGS_BIND;
        struct nlattr *tb[TCA_GACT_MAX + 1];
        struct tcf_chain *goto_ch = NULL;
        struct tc_gact *parm;
        struct tcf_gact *gact;
        int ret = 0;
        u32 index;
        int err;
#ifdef CONFIG_GACT_PROB
        struct tc_gact_p *p_parm = NULL;
#endif

        if (nla == NULL)
                return -EINVAL;

        err = nla_parse_nested_deprecated(tb, TCA_GACT_MAX, nla, gact_policy,
                                          NULL);
        if (err < 0)
                return err;

        if (tb[TCA_GACT_PARMS] == NULL)
                return -EINVAL;
        parm = nla_data(tb[TCA_GACT_PARMS]);
        index = parm->index;

#ifndef CONFIG_GACT_PROB
        if (tb[TCA_GACT_PROB] != NULL)
                return -EOPNOTSUPP;
#else
        if (tb[TCA_GACT_PROB]) {
                p_parm = nla_data(tb[TCA_GACT_PROB]);
                if (p_parm->ptype >= MAX_RAND)
                        return -EINVAL;
                if (!tcf_action_valid(p_parm->paction)) {
                        NL_SET_ERR_MSG(extack,
                                       "invalid fallback control action");
                        return -EINVAL;
                }
                if (TC_ACT_EXT_CMP(p_parm->paction, TC_ACT_GOTO_CHAIN)) {
                        NL_SET_ERR_MSG(extack,
                                       "goto chain not allowed on fallback");
                        return -EINVAL;
                }
        }
#endif

        err = tcf_idr_check_alloc(tn, &index, a, bind);
        if (!err) {
                ret = tcf_idr_create_from_flags(tn, index, est, a,
                                                &act_gact_ops, bind, flags);
                if (ret) {
                        tcf_idr_cleanup(tn, index);
                        return ret;
                }
                ret = ACT_P_CREATED;
        } else if (err > 0) {
                if (bind)/* dont override defaults */
                        return ACT_P_BOUND;
                if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
                        tcf_idr_release(*a, bind);
                        return -EEXIST;
                }
        } else {
                return err;
        }

        err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
        if (err < 0)
                goto release_idr;
        gact = to_gact(*a);

        spin_lock_bh(&gact->tcf_lock);
        goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
#ifdef CONFIG_GACT_PROB
        if (p_parm) {
                gact->tcfg_paction = p_parm->paction;
                gact->tcfg_pval    = max_t(u16, 1, p_parm->pval);
                /* Make sure tcfg_pval is written before tcfg_ptype
                 * coupled with smp_rmb() in gact_net_rand() & gact_determ()
                 */
                smp_wmb();
                gact->tcfg_ptype   = p_parm->ptype;
        }
#endif
        spin_unlock_bh(&gact->tcf_lock);

        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);

        return ret;
release_idr:
        tcf_idr_release(*a, bind);
        return err;
}

TC_INDIRECT_SCOPE int tcf_gact_act(struct sk_buff *skb,
                                   const struct tc_action *a,
                                   struct tcf_result *res)
{
        struct tcf_gact *gact = to_gact(a);
        int action = READ_ONCE(gact->tcf_action);

#ifdef CONFIG_GACT_PROB
        {
        u32 ptype = READ_ONCE(gact->tcfg_ptype);

        if (ptype)
                action = gact_rand[ptype](gact);
        }
#endif
        tcf_action_update_bstats(&gact->common, skb);
        if (action == TC_ACT_SHOT)
                tcf_action_inc_drop_qstats(&gact->common);

        tcf_lastuse_update(&gact->tcf_tm);

        return action;
}

static void tcf_gact_stats_update(struct tc_action *a, u64 bytes, u64 packets,
                                  u64 drops, u64 lastuse, bool hw)
{
        struct tcf_gact *gact = to_gact(a);
        int action = READ_ONCE(gact->tcf_action);
        struct tcf_t *tm = &gact->tcf_tm;

        tcf_action_update_stats(a, bytes, packets,
                                action == TC_ACT_SHOT ? packets : drops, hw);
        tm->lastuse = max_t(u64, tm->lastuse, lastuse);
}

static int tcf_gact_dump(struct sk_buff *skb, struct tc_action *a,
                         int bind, int ref)
{
        unsigned char *b = skb_tail_pointer(skb);
        struct tcf_gact *gact = to_gact(a);
        struct tc_gact opt = {
                .index   = gact->tcf_index,
                .refcnt  = refcount_read(&gact->tcf_refcnt) - ref,
                .bindcnt = atomic_read(&gact->tcf_bindcnt) - bind,
        };
        struct tcf_t t;

        spin_lock_bh(&gact->tcf_lock);
        opt.action = gact->tcf_action;
        if (nla_put(skb, TCA_GACT_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;
#ifdef CONFIG_GACT_PROB
        if (gact->tcfg_ptype) {
                struct tc_gact_p p_opt = {
                        .paction = gact->tcfg_paction,
                        .pval    = gact->tcfg_pval,
                        .ptype   = gact->tcfg_ptype,
                };

                if (nla_put(skb, TCA_GACT_PROB, sizeof(p_opt), &p_opt))
                        goto nla_put_failure;
        }
#endif
        tcf_tm_dump(&t, &gact->tcf_tm);
        if (nla_put_64bit(skb, TCA_GACT_TM, sizeof(t), &t, TCA_GACT_PAD))
                goto nla_put_failure;
        spin_unlock_bh(&gact->tcf_lock);

        return skb->len;

nla_put_failure:
        spin_unlock_bh(&gact->tcf_lock);
        nlmsg_trim(skb, b);
        return -1;
}

static size_t tcf_gact_get_fill_size(const struct tc_action *act)
{
        size_t sz = nla_total_size(sizeof(struct tc_gact)); /* TCA_GACT_PARMS */

#ifdef CONFIG_GACT_PROB
        if (to_gact(act)->tcfg_ptype)
                /* TCA_GACT_PROB */
                sz += nla_total_size(sizeof(struct tc_gact_p));
#endif

        return sz;
}

static int tcf_gact_offload_act_setup(struct tc_action *act, void *entry_data,
                                      u32 *index_inc, bool bind,
                                      struct netlink_ext_ack *extack)
{
        if (bind) {
                struct flow_action_entry *entry = entry_data;

                if (is_tcf_gact_ok(act)) {
                        entry->id = FLOW_ACTION_ACCEPT;
                } else if (is_tcf_gact_shot(act)) {
                        entry->id = FLOW_ACTION_DROP;
                } else if (is_tcf_gact_trap(act)) {
                        entry->id = FLOW_ACTION_TRAP;
                } else if (is_tcf_gact_goto_chain(act)) {
                        entry->id = FLOW_ACTION_GOTO;
                        entry->chain_index = tcf_gact_goto_chain_index(act);
                } else if (is_tcf_gact_continue(act)) {
                        NL_SET_ERR_MSG_MOD(extack, "Offload of \"continue\" action is not supported");
                        return -EOPNOTSUPP;
                } else if (is_tcf_gact_reclassify(act)) {
                        NL_SET_ERR_MSG_MOD(extack, "Offload of \"reclassify\" action is not supported");
                        return -EOPNOTSUPP;
                } else if (is_tcf_gact_pipe(act)) {
                        NL_SET_ERR_MSG_MOD(extack, "Offload of \"pipe\" action is not supported");
                        return -EOPNOTSUPP;
                } else {
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported generic action offload");
                        return -EOPNOTSUPP;
                }
                *index_inc = 1;
        } else {
                struct flow_offload_action *fl_action = entry_data;

                if (is_tcf_gact_ok(act))
                        fl_action->id = FLOW_ACTION_ACCEPT;
                else if (is_tcf_gact_shot(act))
                        fl_action->id = FLOW_ACTION_DROP;
                else if (is_tcf_gact_trap(act))
                        fl_action->id = FLOW_ACTION_TRAP;
                else if (is_tcf_gact_goto_chain(act))
                        fl_action->id = FLOW_ACTION_GOTO;
                else
                        return -EOPNOTSUPP;
        }

        return 0;
}

static struct tc_action_ops act_gact_ops = {
        .kind                =        "gact",
        .id                =        TCA_ID_GACT,
        .owner                =        THIS_MODULE,
        .act                =        tcf_gact_act,
        .stats_update        =        tcf_gact_stats_update,
        .dump                =        tcf_gact_dump,
        .init                =        tcf_gact_init,
        .get_fill_size        =        tcf_gact_get_fill_size,
        .offload_act_setup =        tcf_gact_offload_act_setup,
        .size                =        sizeof(struct tcf_gact),
};
MODULE_ALIAS_NET_ACT("gact");

static __net_init int gact_init_net(struct net *net)
{
        struct tc_action_net *tn = net_generic(net, act_gact_ops.net_id);

        return tc_action_net_init(net, tn, &act_gact_ops);
}

static void __net_exit gact_exit_net(struct list_head *net_list)
{
        tc_action_net_exit(net_list, act_gact_ops.net_id);
}

static struct pernet_operations gact_net_ops = {
        .init = gact_init_net,
        .exit_batch = gact_exit_net,
        .id   = &act_gact_ops.net_id,
        .size = sizeof(struct tc_action_net),
};

MODULE_AUTHOR("Jamal Hadi Salim(2002-4)");
MODULE_DESCRIPTION("Generic Classifier actions");
MODULE_LICENSE("GPL");

static int __init gact_init_module(void)
{
#ifdef CONFIG_GACT_PROB
        pr_info("GACT probability on\n");
#else
        pr_info("GACT probability NOT on\n");
#endif

        return tcf_register_action(&act_gact_ops, &gact_net_ops);
}

static void __exit gact_cleanup_module(void)
{
        tcf_unregister_action(&act_gact_ops, &gact_net_ops);
}

module_init(gact_init_module);
module_exit(gact_cleanup_module);

























































    8 












    8 

























    8 





































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * GHASH and POLYVAL, x86_64 optimized
 *
 * Copyright 2025 Google LLC
 */
#include <asm/fpu/api.h>
#include <linux/cpufeature.h>

#define NUM_H_POWERS 8

static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx);

asmlinkage void polyval_mul_pclmul(struct polyval_elem *a,
                                   const struct polyval_elem *b);
asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a,
                                       const struct polyval_elem *b);

asmlinkage void ghash_blocks_pclmul(struct polyval_elem *acc,
                                    const struct polyval_elem *key,
                                    const u8 *data, size_t nblocks);
asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc,
                                          const struct polyval_key *key,
                                          const u8 *data, size_t nblocks);

#define polyval_preparekey_arch polyval_preparekey_arch
static void polyval_preparekey_arch(struct polyval_key *key,
                                    const u8 raw_key[POLYVAL_BLOCK_SIZE])
{
        static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
        memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
        if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
                kernel_fpu_begin();
                for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
                        key->h_powers[i] = key->h_powers[i + 1];
                        polyval_mul_pclmul_avx(
                                &key->h_powers[i],
                                &key->h_powers[NUM_H_POWERS - 1]);
                }
                kernel_fpu_end();
        } else {
                for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
                        key->h_powers[i] = key->h_powers[i + 1];
                        polyval_mul_generic(&key->h_powers[i],
                                            &key->h_powers[NUM_H_POWERS - 1]);
                }
        }
}

static void polyval_mul_x86(struct polyval_elem *a,
                            const struct polyval_elem *b)
{
        if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) {
                kernel_fpu_begin();
                if (static_branch_likely(&have_pclmul_avx))
                        polyval_mul_pclmul_avx(a, b);
                else
                        polyval_mul_pclmul(a, b);
                kernel_fpu_end();
        } else {
                polyval_mul_generic(a, b);
        }
}

#define ghash_mul_arch ghash_mul_arch
static void ghash_mul_arch(struct polyval_elem *acc,
                           const struct ghash_key *key)
{
        polyval_mul_x86(acc, &key->h);
}

#define polyval_mul_arch polyval_mul_arch
static void polyval_mul_arch(struct polyval_elem *acc,
                             const struct polyval_key *key)
{
        polyval_mul_x86(acc, &key->h_powers[NUM_H_POWERS - 1]);
}

#define ghash_blocks_arch ghash_blocks_arch
static void ghash_blocks_arch(struct polyval_elem *acc,
                              const struct ghash_key *key,
                              const u8 *data, size_t nblocks)
{
        if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) {
                do {
                        /* Allow rescheduling every 4 KiB. */
                        size_t n = min_t(size_t, nblocks,
                                         4096 / GHASH_BLOCK_SIZE);

                        kernel_fpu_begin();
                        ghash_blocks_pclmul(acc, &key->h, data, n);
                        kernel_fpu_end();
                        data += n * GHASH_BLOCK_SIZE;
                        nblocks -= n;
                } while (nblocks);
        } else {
                ghash_blocks_generic(acc, &key->h, data, nblocks);
        }
}

#define polyval_blocks_arch polyval_blocks_arch
static void polyval_blocks_arch(struct polyval_elem *acc,
                                const struct polyval_key *key,
                                const u8 *data, size_t nblocks)
{
        if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
                do {
                        /* Allow rescheduling every 4 KiB. */
                        size_t n = min_t(size_t, nblocks,
                                         4096 / POLYVAL_BLOCK_SIZE);

                        kernel_fpu_begin();
                        polyval_blocks_pclmul_avx(acc, key, data, n);
                        kernel_fpu_end();
                        data += n * POLYVAL_BLOCK_SIZE;
                        nblocks -= n;
                } while (nblocks);
        } else {
                polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
                                       data, nblocks);
        }
}

#define gf128hash_mod_init_arch gf128hash_mod_init_arch
static void gf128hash_mod_init_arch(void)
{
        if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
                static_branch_enable(&have_pclmul);
                if (boot_cpu_has(X86_FEATURE_AVX))
                        static_branch_enable(&have_pclmul_avx);
        }
}





















   48 




   48 











   48 







   48 

















    2 










   15 





   22 





   14 






   15 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
// SPDX-License-Identifier: GPL-2.0-or-later
/* RxRPC Tx data buffering.
 *
 * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/slab.h>
#include "ar-internal.h"

static atomic_t rxrpc_txbuf_debug_ids;
atomic_t rxrpc_nr_txbuf;

/*
 * Allocate and partially initialise a data transmission buffer.
 */
struct rxrpc_txbuf *rxrpc_alloc_data_txbuf(struct rxrpc_call *call, size_t data_size,
                                           size_t data_align, gfp_t gfp)
{
        struct rxrpc_txbuf *txb;
        size_t total, doff, jsize = sizeof(struct rxrpc_jumbo_header);
        void *buf;

        txb = kzalloc_obj(*txb, gfp);
        if (!txb)
                return NULL;

        /* We put a jumbo header in the buffer, but not a full wire header to
         * avoid delayed-corruption problems with zerocopy.
         */
        doff = round_up(jsize, data_align);
        total = doff + data_size;

        data_align = umax(data_align, L1_CACHE_BYTES);
        mutex_lock(&call->conn->tx_data_alloc_lock);
        buf = page_frag_alloc_align(&call->conn->tx_data_alloc, total, gfp,
                                    data_align);
        mutex_unlock(&call->conn->tx_data_alloc_lock);
        if (!buf) {
                kfree(txb);
                return NULL;
        }

        refcount_set(&txb->ref, 1);
        txb->call_debug_id        = call->debug_id;
        txb->debug_id                = atomic_inc_return(&rxrpc_txbuf_debug_ids);
        txb->alloc_size                = data_size;
        txb->space                = data_size;
        txb->offset                = 0;
        txb->flags                = call->conn->out_clientflag;
        txb->seq                = call->send_top + 1;
        txb->data                = buf + doff;

        trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, 1,
                          rxrpc_txbuf_alloc_data);

        atomic_inc(&rxrpc_nr_txbuf);
        return txb;
}

void rxrpc_see_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
{
        int r = refcount_read(&txb->ref);

        trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, r, what);
}

static void rxrpc_free_txbuf(struct rxrpc_txbuf *txb)
{
        trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, 0,
                          rxrpc_txbuf_free);
        if (txb->data)
                page_frag_free(txb->data);
        kfree(txb);
        atomic_dec(&rxrpc_nr_txbuf);
}

void rxrpc_put_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
{
        unsigned int debug_id, call_debug_id;
        rxrpc_seq_t seq;
        bool dead;
        int r;

        if (txb) {
                debug_id = txb->debug_id;
                call_debug_id = txb->call_debug_id;
                seq = txb->seq;
                dead = __refcount_dec_and_test(&txb->ref, &r);
                trace_rxrpc_txbuf(debug_id, call_debug_id, seq, r - 1, what);
                if (dead)
                        rxrpc_free_txbuf(txb);
        }
}




















 5613 












 5163 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_BH_H
#define _LINUX_BH_H

#include <linux/instruction_pointer.h>
#include <linux/preempt.h>

#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_TRACE_IRQFLAGS)
extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt);
#else
static __always_inline void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
{
        preempt_count_add(cnt);
        barrier();
}
#endif

static inline void local_bh_disable(void)
{
        __local_bh_disable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET);
}

extern void _local_bh_enable(void);
extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt);

static inline void local_bh_enable_ip(unsigned long ip)
{
        __local_bh_enable_ip(ip, SOFTIRQ_DISABLE_OFFSET);
}

static inline void local_bh_enable(void)
{
        __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET);
}

#ifdef CONFIG_PREEMPT_RT
extern bool local_bh_blocked(void);
#else
static inline bool local_bh_blocked(void) { return false; }
#endif

#endif /* _LINUX_BH_H */


































































































   13 
   13 

   13 








   33 



   33 




















  176 














    7 


    1 





  162 


    3 

    4 


    2 







  143 

    4 


   23 




    4 


  144 
    6 




    2 


   16 



   21 



  163 

















    2 








   61 
   11 











   31 
    6 






   62 

















   20 
   86 


   91 
   91 









   91 



   70 






   14 



   77 













   76 
    7 








    1 


   15 


   66 




    1 




    2 





   80 

























































    2 


    2 










    2 


    2 





































































































































   34 









   34 







   34 



   34 





   34 


   34 


   34 


   34 
   34 

   34 









   29 









   33 





   55 




   55 





   25 








    1 


   24 


    1 








    7 






   20 



    2 

    2 












    2 


    4 



    2 



    2 





















   10 





































    8 
    8 




    7 


    1 



























   24 

   18 

   42 














   42 
   42 




   42 


































   14 
   14 


   14 









   14 





   14 
   14 













   14 








   14 





   14 






   13 
    1 


   14 































   14 



    9 

    5 





   10 

    4 





    4 
    4 

   14 










   14 


   14 






    4 




   14 




   14 



    3 




   27 
   27 


















    2 





   21 


   22 

    7 


   15 


   15 





    4 
    4 


   10 

    1 












    1 



















    4 

   10 


















    4 
    4 




    4 





   10 




   14 










   15 


























































































































































   49 

   45 
    4 




















   57 

   21 

   36 



    1 



   34 



    1 




    2 





































































































































































































































  101 
















  101 





















   97 
    4 




    2 
   19 

   21 
   21 
   21 
   11 
   11 












   89 



   49 



   49 







   39 


    4 

   84 




    4 

    5 





    1 















    2 
    4 
    3 

    3 


    1 
    2 
    3 


    2 


    1 










   77 

















   77 



   91 
   91 


   91 









    1 







   90 





    2 


   87 


   86 






   84 


   79 





   58 
















    3 
    1 













    2 











    2 
    2 
    2 

































   55 

   11 







    6 





    5 
















   50 



    3 





    3 







   50 
    3 




    3 




    1 







   22 



   43 





   11 
    2 



























































































































   54 










  154 

























































































































































































   25 













































































   25 





   25 


























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        TCP over IPv6
 *        Linux INET6 implementation
 *
 *        Authors:
 *        Pedro Roque                <roque@di.fc.ul.pt>
 *
 *        Based on:
 *        linux/net/ipv4/tcp.c
 *        linux/net/ipv4/tcp_input.c
 *        linux/net/ipv4/tcp_output.c
 *
 *        Fixes:
 *        Hideaki YOSHIFUJI        :        sin6_scope_id support
 *        YOSHIFUJI Hideaki @USAGI and:        Support IPV6_V6ONLY socket option, which
 *        Alexey Kuznetsov                allow both IPv4 and IPv6 sockets to bind
 *                                        a single port at the same time.
 *        YOSHIFUJI Hideaki @USAGI:        convert /proc/net/tcp6 to seq_file.
 */

#include <linux/bottom_half.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/jiffies.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/netdevice.h>
#include <linux/init.h>
#include <linux/jhash.h>
#include <linux/ipsec.h>
#include <linux/times.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/ipv6.h>
#include <linux/icmpv6.h>
#include <linux/random.h>
#include <linux/indirect_call_wrapper.h>

#include <net/aligned_data.h>
#include <net/tcp.h>
#include <net/ndisc.h>
#include <net/inet6_hashtables.h>
#include <net/inet6_connection_sock.h>
#include <net/ipv6.h>
#include <net/transp_v6.h>
#include <net/addrconf.h>
#include <net/ip6_route.h>
#include <net/ip6_checksum.h>
#include <net/inet_ecn.h>
#include <net/protocol.h>
#include <net/xfrm.h>
#include <net/snmp.h>
#include <net/dsfield.h>
#include <net/timewait_sock.h>
#include <net/inet_common.h>
#include <net/secure_seq.h>
#include <net/hotdata.h>
#include <net/busy_poll.h>
#include <net/rstreason.h>
#include <net/psp.h>

#include <linux/proc_fs.h>
#include <linux/seq_file.h>

#include <crypto/md5.h>
#include <crypto/utils.h>

#include <trace/events/tcp.h>

static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
                              enum sk_rst_reason reason);
static void        tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
                                      struct request_sock *req);

INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);

static const struct inet_connection_sock_af_ops ipv6_mapped;
const struct inet_connection_sock_af_ops ipv6_specific;
#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
#endif

/* Helper returning the inet6 address from a given tcp socket.
 * It can be used in TCP stack instead of inet6_sk(sk).
 * This avoids a dereference and allow compiler optimizations.
 * It is a specialized version of inet6_sk_generic().
 */
#define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \
                                              struct tcp6_sock, tcp)->inet6)

static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
{
        struct dst_entry *dst = skb_dst(skb);

        if (dst && dst_hold_safe(dst)) {
                rcu_assign_pointer(sk->sk_rx_dst, dst);
                sk->sk_rx_dst_ifindex = skb->skb_iif;
                sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst));
        }
}

INDIRECT_CALLABLE_SCOPE union tcp_seq_and_ts_off
tcp_v6_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb)
{
        return secure_tcpv6_seq_and_ts_off(net,
                                           ipv6_hdr(skb)->daddr.s6_addr32,
                                           ipv6_hdr(skb)->saddr.s6_addr32,
                                           tcp_hdr(skb)->dest,
                                           tcp_hdr(skb)->source);
}

static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
                              int addr_len)
{
        /* This check is replicated from tcp_v6_connect() and intended to
         * prevent BPF program called below from accessing bytes that are out
         * of the bound specified by user in addr_len.
         */
        if (addr_len < SIN6_LEN_RFC2133)
                return -EINVAL;

        sock_owned_by_me(sk);

        return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len);
}

static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
                          int addr_len)
{
        struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
        struct inet_connection_sock *icsk = inet_csk(sk);
        struct inet_timewait_death_row *tcp_death_row;
        struct ipv6_pinfo *np = tcp_inet6_sk(sk);
        struct in6_addr *saddr = NULL, *final_p;
        struct inet_sock *inet = inet_sk(sk);
        struct tcp_sock *tp = tcp_sk(sk);
        struct net *net = sock_net(sk);
        struct ipv6_txoptions *opt;
        struct dst_entry *dst;
        struct flowi6 *fl6;
        int addr_type;
        int err;

        if (addr_len < SIN6_LEN_RFC2133)
                return -EINVAL;

        if (usin->sin6_family != AF_INET6)
                return -EAFNOSUPPORT;

        fl6 = &inet_sk(sk)->cork.fl.u.ip6;
        memset(fl6, 0, sizeof(*fl6));

        if (inet6_test_bit(SNDFLOW, sk)) {
                fl6->flowlabel = usin->sin6_flowinfo & IPV6_FLOWINFO_MASK;
                IP6_ECN_flow_init(fl6->flowlabel);
                if (fl6->flowlabel & IPV6_FLOWLABEL_MASK) {
                        struct ip6_flowlabel *flowlabel;
                        flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
                        if (IS_ERR(flowlabel))
                                return -EINVAL;
                        fl6_sock_release(flowlabel);
                }
        }

        /*
         *        connect() to INADDR_ANY means loopback (BSD'ism).
         */

        if (ipv6_addr_any(&usin->sin6_addr)) {
                if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
                        ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
                                               &usin->sin6_addr);
                else
                        usin->sin6_addr = in6addr_loopback;
        }

        addr_type = ipv6_addr_type(&usin->sin6_addr);

        if (addr_type & IPV6_ADDR_MULTICAST)
                return -ENETUNREACH;

        if (addr_type&IPV6_ADDR_LINKLOCAL) {
                if (addr_len >= sizeof(struct sockaddr_in6) &&
                    usin->sin6_scope_id) {
                        /* If interface is set while binding, indices
                         * must coincide.
                         */
                        if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id))
                                return -EINVAL;

                        sk->sk_bound_dev_if = usin->sin6_scope_id;
                }

                /* Connect to link-local address requires an interface */
                if (!sk->sk_bound_dev_if)
                        return -EINVAL;
        }

        if (tp->rx_opt.ts_recent_stamp &&
            !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) {
                tp->rx_opt.ts_recent = 0;
                tp->rx_opt.ts_recent_stamp = 0;
                WRITE_ONCE(tp->write_seq, 0);
        }

        sk->sk_v6_daddr = usin->sin6_addr;
        np->flow_label = fl6->flowlabel;

        /*
         *        TCP over IPv4
         */

        if (addr_type & IPV6_ADDR_MAPPED) {
                u32 exthdrlen = icsk->icsk_ext_hdr_len;
                struct sockaddr_in sin;

                if (ipv6_only_sock(sk))
                        return -ENETUNREACH;

                sin.sin_family = AF_INET;
                sin.sin_port = usin->sin6_port;
                sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];

                /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
                WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
                if (sk_is_mptcp(sk))
                        mptcpv6_handle_mapped(sk, true);
                sk->sk_backlog_rcv = tcp_v4_do_rcv;
#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
                tp->af_specific = &tcp_sock_ipv6_mapped_specific;
#endif

                err = tcp_v4_connect(sk, (struct sockaddr_unsized *)&sin, sizeof(sin));

                if (err) {
                        icsk->icsk_ext_hdr_len = exthdrlen;
                        /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
                        WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
                        if (sk_is_mptcp(sk))
                                mptcpv6_handle_mapped(sk, false);
                        sk->sk_backlog_rcv = tcp_v6_do_rcv;
#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
                        tp->af_specific = &tcp_sock_ipv6_specific;
#endif
                        goto failure;
                }
                np->saddr = sk->sk_v6_rcv_saddr;

                return err;
        }

        if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
                saddr = &sk->sk_v6_rcv_saddr;

        sk_set_txhash(sk);

        fl6->flowi6_proto = IPPROTO_TCP;
        fl6->daddr = sk->sk_v6_daddr;
        fl6->saddr = saddr ? *saddr : np->saddr;
        fl6->flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label);
        fl6->flowi6_oif = sk->sk_bound_dev_if;
        fl6->flowi6_mark = sk->sk_mark;
        fl6->fl6_dport = usin->sin6_port;
        fl6->fl6_sport = inet->inet_sport;
        if (IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) && !fl6->fl6_sport)
                fl6->flowi6_flags = FLOWI_FLAG_ANY_SPORT;
        fl6->flowi6_uid = sk_uid(sk);

        opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
        final_p = fl6_update_dst(fl6, opt, &np->final);

        security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6));

        /* Non-zero mp_hash bypasses rt6_multipath_hash() in
         * fib6_select_path(), letting txhash control ECMP path
         * selection so that sk_rethink_txhash() rehashes onto a
         * different path.  Policies 1-3 derive a deterministic
         * hash from the flow keys and must not be overridden.
         */
        ip6_ecmp_set_mp_hash(net, fl6, sk->sk_txhash);

        dst = ip6_dst_lookup_flow(net, sk, fl6, final_p);
        if (IS_ERR(dst)) {
                err = PTR_ERR(dst);
                goto failure;
        }

        tp->tcp_usec_ts = dst_tcp_usec_ts(dst);
        tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;

        if (!saddr) {
                saddr = &fl6->saddr;

                err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
                if (err) {
                        dst_release(dst);
                        goto failure;
                }
        }

        /* set the source address */
        np->saddr = *saddr;
        inet->inet_rcv_saddr = LOOPBACK4_IPV6;

        sk->sk_gso_type = SKB_GSO_TCPV6;
        ip6_dst_store(sk, dst, false, false);

        icsk->icsk_ext_hdr_len = psp_sk_overhead(sk);
        if (opt)
                icsk->icsk_ext_hdr_len += opt->opt_flen +
                                          opt->opt_nflen;

        tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);

        inet->inet_dport = usin->sin6_port;

        tcp_set_state(sk, TCP_SYN_SENT);
        err = inet6_hash_connect(tcp_death_row, sk);
        if (err)
                goto late_failure;

        if (likely(!tp->repair)) {
                union tcp_seq_and_ts_off st;

                st = secure_tcpv6_seq_and_ts_off(net,
                                                 np->saddr.s6_addr32,
                                                 sk->sk_v6_daddr.s6_addr32,
                                                 inet->inet_sport,
                                                 inet->inet_dport);
                if (!tp->write_seq)
                        WRITE_ONCE(tp->write_seq, st.seq);
                WRITE_ONCE(tp->tsoffset, st.ts_off);
        }

        if (tcp_fastopen_defer_connect(sk, &err))
                return err;
        if (err)
                goto late_failure;

        err = tcp_connect(sk);
        if (err)
                goto late_failure;

        return 0;

late_failure:
        tcp_set_state(sk, TCP_CLOSE);
        inet_bhash2_reset_saddr(sk);
failure:
        inet->inet_dport = 0;
        sk->sk_route_caps = 0;
        return err;
}

static struct dst_entry *inet6_csk_update_pmtu(struct sock *sk, u32 mtu)
{
        struct flowi6 *fl6 = &inet_sk(sk)->cork.fl.u.ip6;
        struct dst_entry *dst;

        dst = inet6_csk_route_socket(sk, fl6);

        if (IS_ERR(dst))
                return NULL;
        dst->ops->update_pmtu(dst, sk, NULL, mtu, true);

        dst = inet6_csk_route_socket(sk, fl6);
        return IS_ERR(dst) ? NULL : dst;
}

static void tcp_v6_mtu_reduced(struct sock *sk)
{
        struct dst_entry *dst;
        u32 mtu, dmtu;

        if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
                return;

        mtu = READ_ONCE(tcp_sk(sk)->mtu_info);

        /* Drop requests trying to increase our current mss.
         * Check done in __ip6_rt_update_pmtu() is too late.
         */
        if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache)
                return;

        dst = inet6_csk_update_pmtu(sk, mtu);
        if (!dst)
                return;

        dmtu = dst6_mtu(dst);
        if (inet_csk(sk)->icsk_pmtu_cookie > dmtu) {
                tcp_sync_mss(sk, dmtu);
                tcp_simple_retransmit(sk);
        }
}

static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                u8 type, u8 code, int offset, __be32 info)
{
        const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
        const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
        struct net *net = dev_net_rcu(skb->dev);
        struct request_sock *fastopen;
        struct ipv6_pinfo *np;
        struct tcp_sock *tp;
        __u32 seq, snd_una;
        struct sock *sk;
        bool fatal;
        int err;

        sk = __inet6_lookup_established(net, &hdr->daddr, th->dest,
                                        &hdr->saddr, ntohs(th->source),
                                        skb->dev->ifindex, inet6_sdif(skb));

        if (!sk) {
                __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
                                  ICMP6_MIB_INERRORS);
                return -ENOENT;
        }

        if (sk->sk_state == TCP_TIME_WAIT) {
                /* To increase the counter of ignored icmps for TCP-AO */
                tcp_ao_ignore_icmp(sk, AF_INET6, type, code);
                inet_twsk_put(inet_twsk(sk));
                return 0;
        }
        seq = ntohl(th->seq);
        fatal = icmpv6_err_convert(type, code, &err);
        if (sk->sk_state == TCP_NEW_SYN_RECV) {
                tcp_req_err(sk, seq, fatal);
                return 0;
        }

        if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) {
                sock_put(sk);
                return 0;
        }

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
                __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);

        if (sk->sk_state == TCP_CLOSE)
                goto out;

        if (static_branch_unlikely(&ip6_min_hopcount)) {
                /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
                if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) {
                        __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
                        goto out;
                }
        }

        tp = tcp_sk(sk);
        /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
        fastopen = rcu_dereference(tp->fastopen_rsk);
        snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
        if (sk->sk_state != TCP_LISTEN &&
            !between(seq, snd_una, tp->snd_nxt)) {
                __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
                goto out;
        }

        np = tcp_inet6_sk(sk);

        if (type == NDISC_REDIRECT) {
                if (!sock_owned_by_user(sk)) {
                        struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie);

                        if (dst)
                                dst->ops->redirect(dst, sk, skb);
                }
                goto out;
        }

        if (type == ICMPV6_PKT_TOOBIG) {
                u32 mtu = ntohl(info);

                /* We are not interested in TCP_LISTEN and open_requests
                 * (SYN-ACKs send out by Linux are always <576bytes so
                 * they should go through unfragmented).
                 */
                if (sk->sk_state == TCP_LISTEN)
                        goto out;

                if (!ip6_sk_accept_pmtu(sk))
                        goto out;

                if (mtu < IPV6_MIN_MTU)
                        goto out;

                WRITE_ONCE(tp->mtu_info, mtu);

                if (!sock_owned_by_user(sk))
                        tcp_v6_mtu_reduced(sk);
                else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
                                           &sk->sk_tsq_flags))
                        sock_hold(sk);
                goto out;
        }


        /* Might be for an request_sock */
        switch (sk->sk_state) {
        case TCP_SYN_SENT:
        case TCP_SYN_RECV:
                /* Only in fast or simultaneous open. If a fast open socket is
                 * already accepted it is treated as a connected one below.
                 */
                if (fastopen && !fastopen->sk)
                        break;

                ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);

                if (!sock_owned_by_user(sk))
                        tcp_done_with_error(sk, err);
                else
                        WRITE_ONCE(sk->sk_err_soft, err);
                goto out;
        case TCP_LISTEN:
                break;
        default:
                /* check if this ICMP message allows revert of backoff.
                 * (see RFC 6069)
                 */
                if (!fastopen && type == ICMPV6_DEST_UNREACH &&
                    code == ICMPV6_NOROUTE)
                        tcp_ld_RTO_revert(sk, seq);
        }

        if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) {
                WRITE_ONCE(sk->sk_err, err);
                sk_error_report(sk);
        } else {
                WRITE_ONCE(sk->sk_err_soft, err);
        }
out:
        bh_unlock_sock(sk);
        sock_put(sk);
        return 0;
}


static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
                              struct flowi *fl,
                              struct request_sock *req,
                              struct tcp_fastopen_cookie *foc,
                              enum tcp_synack_type synack_type,
                              struct sk_buff *syn_skb)
{
        struct inet_request_sock *ireq = inet_rsk(req);
        const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
        struct ipv6_txoptions *opt;
        struct flowi6 *fl6 = &fl->u.ip6;
        struct sk_buff *skb;
        int err = -ENOMEM;
        u8 tclass;

        /* First, grab a route. */
        if (!dst && (dst = inet6_csk_route_req(sk, NULL, fl6, req,
                                               IPPROTO_TCP)) == NULL)
                goto done;

        skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);

        if (skb) {
                tcp_rsk(req)->syn_ect_snt = np->tclass & INET_ECN_MASK;
                __tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr,
                                    &ireq->ir_v6_rmt_addr);

                fl6->daddr = ireq->ir_v6_rmt_addr;
                if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts)
                        fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts));

                tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
                                (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
                                (np->tclass & INET_ECN_MASK) :
                                np->tclass;

                if (!INET_ECN_is_capable(tclass) &&
                    tcp_bpf_ca_needs_ecn((struct sock *)req))
                        tclass |= INET_ECN_ECT_0;

                rcu_read_lock();
                opt = ireq->ipv6_opt;
                if (!opt)
                        opt = rcu_dereference(np->opt);
                err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark),
                               opt, tclass, READ_ONCE(sk->sk_priority));
                rcu_read_unlock();
                err = net_xmit_eval(err);
        }

done:
        return err;
}


static void tcp_v6_reqsk_destructor(struct request_sock *req)
{
        kfree(inet_rsk(req)->ipv6_opt);
        consume_skb(inet_rsk(req)->pktopts);
}

#ifdef CONFIG_TCP_MD5SIG
static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
                                                   const struct in6_addr *addr,
                                                   int l3index)
{
        return tcp_md5_do_lookup(sk, l3index,
                                 (union tcp_md5_addr *)addr, AF_INET6);
}

static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
                                                const struct sock *addr_sk)
{
        int l3index;

        l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
                                                 addr_sk->sk_bound_dev_if);
        return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr,
                                    l3index);
}

static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
                                 sockptr_t optval, int optlen)
{
        struct tcp_md5sig cmd;
        struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
        union tcp_ao_addr *addr;
        int l3index = 0;
        u8 prefixlen;
        bool l3flag;
        u8 flags;

        if (optlen < sizeof(cmd))
                return -EINVAL;

        if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
                return -EFAULT;

        if (sin6->sin6_family != AF_INET6)
                return -EINVAL;

        flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
        l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;

        if (optname == TCP_MD5SIG_EXT &&
            cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
                prefixlen = cmd.tcpm_prefixlen;
                if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) &&
                                        prefixlen > 32))
                        return -EINVAL;
        } else {
                prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128;
        }

        if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
            cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
                struct net_device *dev;

                rcu_read_lock();
                dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
                if (dev && netif_is_l3_master(dev))
                        l3index = dev->ifindex;
                rcu_read_unlock();

                /* ok to reference set/not set outside of rcu;
                 * right now device MUST be an L3 master
                 */
                if (!dev || !l3index)
                        return -EINVAL;
        }

        if (!cmd.tcpm_keylen) {
                if (ipv6_addr_v4mapped(&sin6->sin6_addr))
                        return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
                                              AF_INET, prefixlen,
                                              l3index, flags);
                return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
                                      AF_INET6, prefixlen, l3index, flags);
        }

        if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
                return -EINVAL;

        if (ipv6_addr_v4mapped(&sin6->sin6_addr)) {
                addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3];

                /* Don't allow keys for peers that have a matching TCP-AO key.
                 * See the comment in tcp_ao_add_cmd()
                 */
                if (tcp_ao_required(sk, addr, AF_INET,
                                    l3flag ? l3index : -1, false))
                        return -EKEYREJECTED;
                return tcp_md5_do_add(sk, addr,
                                      AF_INET, prefixlen, l3index, flags,
                                      cmd.tcpm_key, cmd.tcpm_keylen);
        }

        addr = (union tcp_md5_addr *)&sin6->sin6_addr;

        /* Don't allow keys for peers that have a matching TCP-AO key.
         * See the comment in tcp_ao_add_cmd()
         */
        if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false))
                return -EKEYREJECTED;

        return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags,
                              cmd.tcpm_key, cmd.tcpm_keylen);
}

static void tcp_v6_md5_hash_headers(struct md5_ctx *ctx,
                                    const struct in6_addr *daddr,
                                    const struct in6_addr *saddr,
                                    const struct tcphdr *th, int nbytes)
{
        struct {
                struct tcp6_pseudohdr ip; /* TCP pseudo-header (RFC2460) */
                struct tcphdr tcp;
        } h;

        h.ip.saddr = *saddr;
        h.ip.daddr = *daddr;
        h.ip.protocol = cpu_to_be32(IPPROTO_TCP);
        h.ip.len = cpu_to_be32(nbytes);
        h.tcp = *th;
        h.tcp.check = 0;
        md5_update(ctx, (const u8 *)&h, sizeof(h.ip) + sizeof(h.tcp));
}

static noinline_for_stack void
tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
                    const struct in6_addr *daddr, struct in6_addr *saddr,
                    const struct tcphdr *th)
{
        struct md5_ctx ctx;

        md5_init(&ctx);
        tcp_v6_md5_hash_headers(&ctx, daddr, saddr, th, th->doff << 2);
        tcp_md5_hash_key(&ctx, key);
        md5_final(&ctx, md5_hash);
}

static noinline_for_stack void
tcp_v6_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
                    const struct sock *sk, const struct sk_buff *skb)
{
        const struct tcphdr *th = tcp_hdr(skb);
        const struct in6_addr *saddr, *daddr;
        struct md5_ctx ctx;

        if (sk) { /* valid for establish/request sockets */
                saddr = &sk->sk_v6_rcv_saddr;
                daddr = &sk->sk_v6_daddr;
        } else {
                const struct ipv6hdr *ip6h = ipv6_hdr(skb);
                saddr = &ip6h->saddr;
                daddr = &ip6h->daddr;
        }

        md5_init(&ctx);
        tcp_v6_md5_hash_headers(&ctx, daddr, saddr, th, skb->len);
        tcp_md5_hash_skb_data(&ctx, skb, th->doff << 2);
        tcp_md5_hash_key(&ctx, key);
        md5_final(&ctx, md5_hash);
}
#endif

static void tcp_v6_init_req(struct request_sock *req,
                            const struct sock *sk_listener,
                            struct sk_buff *skb,
                            u32 tw_isn)
{
        bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
        struct inet_request_sock *ireq = inet_rsk(req);
        const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);

        ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
        ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
        ireq->ir_rmt_addr = LOOPBACK4_IPV6;
        ireq->ir_loc_addr = LOOPBACK4_IPV6;

        /* So that link locals have meaning */
        if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
            ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
                ireq->ir_iif = tcp_v6_iif(skb);

        if (!tw_isn &&
            (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) ||
             np->rxopt.bits.rxinfo ||
             np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
             np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) {
                refcount_inc(&skb->users);
                ireq->pktopts = skb;
        }
}

static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
                                          struct sk_buff *skb,
                                          struct flowi *fl,
                                          struct request_sock *req,
                                          u32 tw_isn)
{
        tcp_v6_init_req(req, sk, skb, tw_isn);

        if (security_inet_conn_request(sk, skb, req))
                return NULL;

        return inet6_csk_route_req(sk, NULL, &fl->u.ip6, req, IPPROTO_TCP);
}

struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
        .family                =        AF_INET6,
        .obj_size        =        sizeof(struct tcp6_request_sock),
        .send_ack        =        tcp_v6_reqsk_send_ack,
        .destructor        =        tcp_v6_reqsk_destructor,
        .send_reset        =        tcp_v6_send_reset,
};

const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
        .mss_clamp        =        IPV6_MIN_MTU - sizeof(struct tcphdr) -
                                sizeof(struct ipv6hdr),
#ifdef CONFIG_TCP_MD5SIG
        .req_md5_lookup        =        tcp_v6_md5_lookup,
        .calc_md5_hash        =        tcp_v6_md5_hash_skb,
#endif
#ifdef CONFIG_TCP_AO
        .ao_lookup        =        tcp_v6_ao_lookup_rsk,
        .ao_calc_key        =        tcp_v6_ao_calc_key_rsk,
        .ao_synack_hash =        tcp_v6_ao_synack_hash,
#endif
#ifdef CONFIG_SYN_COOKIES
        .cookie_init_seq =        cookie_v6_init_sequence,
#endif
        .route_req        =        tcp_v6_route_req,
        .init_seq_and_ts_off        = tcp_v6_init_seq_and_ts_off,
        .send_synack        =        tcp_v6_send_synack,
};

static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
                                 u32 ack, u32 win, u32 tsval, u32 tsecr,
                                 int oif, int rst, u8 tclass, __be32 label,
                                 u32 priority, u32 txhash, struct tcp_key *key)
{
        struct net *net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb);
        unsigned int tot_len = sizeof(struct tcphdr);
        struct sock *ctl_sk = net->ipv6.tcp_sk;
        const struct tcphdr *th = tcp_hdr(skb);
        __be32 mrst = 0, *topt;
        struct dst_entry *dst;
        struct sk_buff *buff;
        struct tcphdr *t1;
        struct flowi6 fl6;
        u32 mark = 0;

        if (tsecr)
                tot_len += TCPOLEN_TSTAMP_ALIGNED;
        if (tcp_key_is_md5(key))
                tot_len += TCPOLEN_MD5SIG_ALIGNED;
        if (tcp_key_is_ao(key))
                tot_len += tcp_ao_len_aligned(key->ao_key);

#ifdef CONFIG_MPTCP
        if (rst && !tcp_key_is_md5(key)) {
                mrst = mptcp_reset_option(skb);

                if (mrst)
                        tot_len += sizeof(__be32);
        }
#endif

        buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
        if (!buff)
                return;

        skb_reserve(buff, MAX_TCP_HEADER);

        t1 = skb_push(buff, tot_len);
        skb_reset_transport_header(buff);

        /* Swap the send and the receive. */
        memset(t1, 0, sizeof(*t1));
        t1->dest = th->source;
        t1->source = th->dest;
        t1->doff = tot_len / 4;
        t1->seq = htonl(seq);
        t1->ack_seq = htonl(ack);
        t1->ack = !rst || !th->ack;
        t1->rst = rst;
        t1->window = htons(win);

        topt = (__be32 *)(t1 + 1);

        if (tsecr) {
                *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
                                (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
                *topt++ = htonl(tsval);
                *topt++ = htonl(tsecr);
        }

        if (mrst)
                *topt++ = mrst;

#ifdef CONFIG_TCP_MD5SIG
        if (tcp_key_is_md5(key)) {
                *topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
                                (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
                tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key,
                                    &ipv6_hdr(skb)->saddr,
                                    &ipv6_hdr(skb)->daddr, t1);
        }
#endif
#ifdef CONFIG_TCP_AO
        if (tcp_key_is_ao(key)) {
                *topt++ = htonl((TCPOPT_AO << 24) |
                                (tcp_ao_len(key->ao_key) << 16) |
                                (key->ao_key->sndid << 8) |
                                (key->rcv_next));
                memset((u8 *)topt + tcp_ao_maclen(key->ao_key), TCPOPT_NOP,
                       tcp_ao_len_aligned(key->ao_key) - tcp_ao_len(key->ao_key));

                tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key,
                                key->traffic_key,
                                (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr,
                                (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr,
                                t1, key->sne);
        }
#endif

        memset(&fl6, 0, sizeof(fl6));
        fl6.daddr = ipv6_hdr(skb)->saddr;
        fl6.saddr = ipv6_hdr(skb)->daddr;
        fl6.flowlabel = label;

        buff->ip_summed = CHECKSUM_PARTIAL;

        __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr);

        fl6.flowi6_proto = IPPROTO_TCP;
        if (rt6_need_strict(&fl6.daddr) && !oif)
                fl6.flowi6_oif = tcp_v6_iif(skb);
        else {
                if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
                        oif = skb->skb_iif;

                fl6.flowi6_oif = oif;
        }

        if (sk) {
                /* unconstify the socket only to attach it to buff with care. */
                skb_set_owner_edemux(buff, (struct sock *)sk);
                psp_reply_set_decrypted(sk, buff);

                if (sk->sk_state == TCP_TIME_WAIT)
                        mark = inet_twsk(sk)->tw_mark;
                else
                        mark = READ_ONCE(sk->sk_mark);
                skb_set_delivery_time(buff, tcp_transmit_time(sk), SKB_CLOCK_MONOTONIC);
        }
        if (txhash) {
                /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
                skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4);

                /* Select the local ECMP path from the connection's txhash,
                 * so a control packet (RST, or ACK from a time-wait socket)
                 * uses the same nexthop as the data.  Only policy 0 uses
                 * mp_hash; policies 1-3 derive a deterministic hash.
                 */
                ip6_ecmp_set_mp_hash(net, &fl6, txhash);
        }
        fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
        fl6.fl6_dport = t1->dest;
        fl6.fl6_sport = t1->source;
        fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
        security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6));

        /* Pass a socket to ip6_dst_lookup either it is for RST
         * Underlying function will use this to retrieve the network
         * namespace
         */
        if (sk && sk->sk_state != TCP_TIME_WAIT)
                dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/
        else
                dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL);
        if (!IS_ERR(dst)) {
                skb_dst_set(buff, dst);
                ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL,
                         tclass, priority);
                TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
                if (rst)
                        TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
                return;
        }

        sk_skb_reason_drop(sk, buff, SKB_DROP_REASON_IP_OUTNOROUTES);
}

static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
                              enum sk_rst_reason reason)
{
        const struct tcphdr *th = tcp_hdr(skb);
        struct ipv6hdr *ipv6h = ipv6_hdr(skb);
        const __u8 *md5_hash_location = NULL;
#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
        bool allocated_traffic_key = false;
#endif
        const struct tcp_ao_hdr *aoh;
        struct tcp_key key = {};
        u32 seq = 0, ack_seq = 0;
        __be32 label = 0;
        u32 priority = 0;
        struct net *net;
        u32 txhash = 0;
        int oif = 0;
#ifdef CONFIG_TCP_MD5SIG
        unsigned char newhash[16];
        struct sock *sk1 = NULL;
#endif

        if (th->rst)
                return;

        /* If sk not NULL, it means we did a successful lookup and incoming
         * route had to be correct. prequeue might have dropped our dst.
         */
        if (!sk && !ipv6_unicast_destination(skb))
                return;

        net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb);
        /* Invalid TCP option size or twice included auth */
        if (tcp_parse_auth_options(th, &md5_hash_location, &aoh))
                return;
#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
        rcu_read_lock();
#endif
#ifdef CONFIG_TCP_MD5SIG
        if (sk && sk_fullsock(sk)) {
                int l3index;

                /* sdif set, means packet ingressed via a device
                 * in an L3 domain and inet_iif is set to it.
                 */
                l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
                key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index);
                if (key.md5_key)
                        key.type = TCP_KEY_MD5;
        } else if (md5_hash_location) {
                int dif = tcp_v6_iif_l3_slave(skb);
                int sdif = tcp_v6_sdif(skb);
                int l3index;

                /*
                 * active side is lost. Try to find listening socket through
                 * source port, and then find md5 key through listening socket.
                 * we are not loose security here:
                 * Incoming packet is checked with md5 hash with finding key,
                 * no RST generated if md5 hash doesn't match.
                 */
                sk1 = inet6_lookup_listener(net, NULL, 0, &ipv6h->saddr, th->source,
                                            &ipv6h->daddr, ntohs(th->source),
                                            dif, sdif);
                if (!sk1)
                        goto out;

                /* sdif set, means packet ingressed via a device
                 * in an L3 domain and dif is set to it.
                 */
                l3index = tcp_v6_sdif(skb) ? dif : 0;

                key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index);
                if (!key.md5_key)
                        goto out;
                key.type = TCP_KEY_MD5;

                tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb);
                if (crypto_memneq(md5_hash_location, newhash, 16))
                        goto out;
        }
#endif

        if (th->ack)
                seq = ntohl(th->ack_seq);
        else
                ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
                          (th->doff << 2);

#ifdef CONFIG_TCP_AO
        if (aoh) {
                int l3index;

                l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
                if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq,
                                         &key.ao_key, &key.traffic_key,
                                         &allocated_traffic_key,
                                         &key.rcv_next, &key.sne))
                        goto out;
                key.type = TCP_KEY_AO;
        }
#endif

        if (sk) {
                oif = sk->sk_bound_dev_if;
                if (sk_fullsock(sk)) {
                        if (inet6_test_bit(REPFLOW, sk))
                                label = ip6_flowlabel(ipv6h);
                        priority = READ_ONCE(sk->sk_priority);
                        txhash = sk->sk_txhash;
                }
                if (sk->sk_state == TCP_TIME_WAIT) {
                        label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
                        priority = inet_twsk(sk)->tw_priority;
                        txhash = inet_twsk(sk)->tw_txhash;
                }
        } else {
                if (READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) &
                    FLOWLABEL_REFLECT_TCP_RESET)
                        label = ip6_flowlabel(ipv6h);
        }

        trace_tcp_send_reset(sk, skb, reason);

        tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1,
                             ipv6_get_dsfield(ipv6h) & ~INET_ECN_MASK,
                             label, priority, txhash,
                             &key);

#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
out:
        if (allocated_traffic_key)
                kfree(key.traffic_key);
        rcu_read_unlock();
#endif
}

static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
                            u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
                            struct tcp_key *key, u8 tclass,
                            __be32 label, u32 priority, u32 txhash)
{
        tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0,
                             tclass, label, priority, txhash, key);
}

static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb,
                                enum tcp_tw_status tw_status)
{
        struct inet_timewait_sock *tw = inet_twsk(sk);
        struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
        u8 tclass = tw->tw_tclass;
        struct tcp_key key = {};

        if (tw_status == TCP_TW_ACK_OOW)
                tclass &= ~INET_ECN_MASK;
#ifdef CONFIG_TCP_AO
        struct tcp_ao_info *ao_info;

        if (static_branch_unlikely(&tcp_ao_needed.key)) {

                /* FIXME: the segment to-be-acked is not verified yet */
                ao_info = rcu_dereference(tcptw->ao_info);
                if (ao_info) {
                        const struct tcp_ao_hdr *aoh;

                        /* Invalid TCP option size or twice included auth */
                        if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
                                goto out;
                        if (aoh)
                                key.ao_key = tcp_ao_established_key(sk, ao_info,
                                                                    aoh->rnext_keyid, -1);
                }
        }
        if (key.ao_key) {
                struct tcp_ao_key *rnext_key;

                key.traffic_key = snd_other_key(key.ao_key);
                /* rcv_next switches to our rcv_next */
                rnext_key = READ_ONCE(ao_info->rnext_key);
                key.rcv_next = rnext_key->rcvid;
                key.sne = READ_ONCE(ao_info->snd_sne);
                key.type = TCP_KEY_AO;
#else
        if (0) {
#endif
#ifdef CONFIG_TCP_MD5SIG
        } else if (static_branch_unlikely(&tcp_md5_needed.key)) {
                key.md5_key = tcp_twsk_md5_key(tcptw);
                if (key.md5_key)
                        key.type = TCP_KEY_MD5;
#endif
        }

        tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt,
                        READ_ONCE(tcptw->tw_rcv_nxt),
                        tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
                        tcp_tw_tsval(tcptw),
                        READ_ONCE(tcptw->tw_ts_recent), tw->tw_bound_dev_if,
                        &key, tclass, cpu_to_be32(tw->tw_flowlabel),
                        tw->tw_priority, tw->tw_txhash);

#ifdef CONFIG_TCP_AO
out:
#endif
        inet_twsk_put(tw);
}

static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
                                  struct request_sock *req)
{
        struct tcp_key key = {};

#ifdef CONFIG_TCP_AO
        if (static_branch_unlikely(&tcp_ao_needed.key) &&
            tcp_rsk_used_ao(req)) {
                const struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
                const struct tcp_ao_hdr *aoh;
                int l3index;

                l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
                /* Invalid TCP option size or twice included auth */
                if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
                        return;
                if (!aoh)
                        return;
                key.ao_key = tcp_ao_do_lookup(sk, l3index,
                                              (union tcp_ao_addr *)addr,
                                              AF_INET6, aoh->rnext_keyid, -1);
                if (unlikely(!key.ao_key)) {
                        /* Send ACK with any matching MKT for the peer */
                        key.ao_key = tcp_ao_do_lookup(sk, l3index,
                                                      (union tcp_ao_addr *)addr,
                                                      AF_INET6, -1, -1);
                        /* Matching key disappeared (user removed the key?)
                         * let the handshake timeout.
                         */
                        if (!key.ao_key) {
                                net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n",
                                                     addr,
                                                     ntohs(tcp_hdr(skb)->source),
                                                     &ipv6_hdr(skb)->daddr,
                                                     ntohs(tcp_hdr(skb)->dest));
                                return;
                        }
                }
                key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
                if (!key.traffic_key)
                        return;

                key.type = TCP_KEY_AO;
                key.rcv_next = aoh->keyid;
                tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
#else
        if (0) {
#endif
#ifdef CONFIG_TCP_MD5SIG
        } else if (static_branch_unlikely(&tcp_md5_needed.key)) {
                int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;

                key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr,
                                                   l3index);
                if (key.md5_key)
                        key.type = TCP_KEY_MD5;
#endif
        }

        /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
         * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
         */
        tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
                        tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
                        tcp_rsk(req)->rcv_nxt,
                        tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale,
                        tcp_rsk_tsval(tcp_rsk(req)),
                        req->ts_recent, sk->sk_bound_dev_if,
                        &key, ipv6_get_dsfield(ipv6_hdr(skb)) & ~INET_ECN_MASK,
                        0,
                        READ_ONCE(sk->sk_priority),
                        READ_ONCE(tcp_rsk(req)->txhash));
        if (tcp_key_is_ao(&key))
                kfree(key.traffic_key);
}


static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
{
#ifdef CONFIG_SYN_COOKIES
        const struct tcphdr *th = tcp_hdr(skb);

        if (!th->syn)
                sk = cookie_v6_check(sk, skb);
#endif
        return sk;
}

u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
                         struct tcphdr *th, u32 *cookie)
{
        u16 mss = 0;
#ifdef CONFIG_SYN_COOKIES
        mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops,
                                    &tcp_request_sock_ipv6_ops, sk, th);
        if (mss) {
                *cookie = __cookie_v6_init_sequence(iph, th, &mss);
                tcp_synq_overflow(sk);
        }
#endif
        return mss;
}

static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
{
        if (skb->protocol == htons(ETH_P_IP))
                return tcp_v4_conn_request(sk, skb);

        if (!ipv6_unicast_destination(skb))
                goto drop;

        if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
                __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
                return 0;
        }

        return tcp_conn_request(&tcp6_request_sock_ops,
                                &tcp_request_sock_ipv6_ops, sk, skb);

drop:
        tcp_listendrop(sk);
        return 0; /* don't send reset */
}

static void tcp_v6_restore_cb(struct sk_buff *skb)
{
        /* We need to move header back to the beginning if xfrm6_policy_check()
         * and tcp_v6_fill_cb() are going to be called again.
         * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
         */
        memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
                sizeof(struct inet6_skb_parm));
}

/* Called from tcp_v4_syn_recv_sock() for v6_mapped children. */
static void tcp_v6_mapped_child_init(struct sock *newsk, const struct sock *sk)
{
        struct inet_sock *newinet = inet_sk(newsk);
        struct ipv6_pinfo *newnp;

        newinet->pinet6 = newnp = tcp_inet6_sk(newsk);
        newinet->ipv6_fl_list = NULL;

        memcpy(newnp, tcp_inet6_sk(sk), sizeof(struct ipv6_pinfo));

        newnp->saddr = newsk->sk_v6_rcv_saddr;

        inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
        if (sk_is_mptcp(newsk))
                mptcpv6_handle_mapped(newsk, true);
        newsk->sk_backlog_rcv = tcp_v4_do_rcv;
#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
        tcp_sk(newsk)->af_specific = &tcp_sock_ipv6_mapped_specific;
#endif

        newnp->ipv6_mc_list = NULL;
        newnp->ipv6_ac_list = NULL;
        newnp->pktoptions  = NULL;
        newnp->opt           = NULL;

        /* tcp_v4_syn_recv_sock() has initialized newinet->mc_{index,ttl} */
        newnp->mcast_oif   = newinet->mc_index;
        newnp->mcast_hops  = newinet->mc_ttl;

        newnp->rcv_flowinfo = 0;
        if (inet6_test_bit(REPFLOW, sk))
                newnp->flow_label = 0;
}

static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
                                         struct request_sock *req,
                                         struct dst_entry *dst,
                                         struct request_sock *req_unhash,
                                         bool *own_req,
                                         void (*opt_child_init)(struct sock *newsk,
                                                                const struct sock *sk))
{
        const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
        struct inet_request_sock *ireq;
        struct ipv6_txoptions *opt;
        struct inet_sock *newinet;
        bool found_dup_sk = false;
        struct ipv6_pinfo *newnp;
        struct tcp_sock *newtp;
        struct sock *newsk;
#ifdef CONFIG_TCP_MD5SIG
        struct tcp_md5sig_key *key;
        int l3index;
#endif
        struct flowi6 fl6;

        if (skb->protocol == htons(ETH_P_IP))
                return tcp_v4_syn_recv_sock(sk, skb, req, dst,
                                            req_unhash, own_req,
                                            tcp_v6_mapped_child_init);
        ireq = inet_rsk(req);

        if (sk_acceptq_is_full(sk))
                goto exit_overflow;

        dst = inet6_csk_route_req(sk, dst, &fl6, req, IPPROTO_TCP);
        if (!dst)
                goto exit;

        newsk = tcp_create_openreq_child(sk, req, skb);
        if (!newsk)
                goto exit_nonewsk;

        /*
         * No need to charge this sock to the relevant IPv6 refcnt debug socks
         * count here, tcp_create_openreq_child now does this for us, see the
         * comment in that function for the gory details. -acme
         */

        newsk->sk_gso_type = SKB_GSO_TCPV6;
        inet6_sk_rx_dst_set(newsk, skb);

        newinet = inet_sk(newsk);
        newinet->cork.fl.u.ip6 = fl6;
        newinet->pinet6 = tcp_inet6_sk(newsk);
        newinet->ipv6_fl_list = NULL;
        newinet->inet_opt = NULL;

        newtp = tcp_sk(newsk);
        newnp = tcp_inet6_sk(newsk);

        memcpy(newnp, np, sizeof(struct ipv6_pinfo));

        ip6_dst_store(newsk, dst, false, false);

        newnp->saddr = ireq->ir_v6_loc_addr;

        /* Now IPv6 options...

           First: no IPv4 options.
         */
        newnp->ipv6_mc_list = NULL;
        newnp->ipv6_ac_list = NULL;

        /* Clone RX bits */
        newnp->rxopt.all = np->rxopt.all;

        newnp->pktoptions = NULL;
        newnp->opt          = NULL;
        newnp->mcast_oif  = tcp_v6_iif(skb);
        newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
        newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb));
        if (inet6_test_bit(REPFLOW, sk))
                newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb));

        /* Set ToS of the new socket based upon the value of incoming SYN.
         * ECT bits are set later in tcp_init_transfer().
         */
        if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
                newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;

        /* Clone native IPv6 options from listening socket (if any)

           Yes, keeping reference count would be much more clever,
           but we make one more one thing there: reattach optmem
           to newsk.
         */
        opt = ireq->ipv6_opt;
        if (!opt)
                opt = rcu_dereference(np->opt);
        if (opt) {
                opt = ipv6_dup_options(newsk, opt);
                RCU_INIT_POINTER(newnp->opt, opt);
        }
        inet_csk(newsk)->icsk_ext_hdr_len = 0;
        if (opt)
                inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
                                                    opt->opt_flen;

        tcp_ca_openreq_child(newsk, dst);

        tcp_sync_mss(newsk, dst6_mtu(dst));
        newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst));

        tcp_initialize_rcv_mss(newsk);

#ifdef CONFIG_TCP_MD5SIG
        l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);

        if (!tcp_rsk_used_ao(req)) {
                /* Copy over the MD5 key from the original socket */
                key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index);
                if (key) {
                        const union tcp_md5_addr *addr;

                        addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr;
                        if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key))
                                goto put_and_exit;
                }
        }
#endif
#ifdef CONFIG_TCP_AO
        /* Copy over tcp_ao_info if any */
        if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6))
                goto put_and_exit; /* OOM */
#endif

        if (__inet_inherit_port(sk, newsk) < 0)
                goto put_and_exit;
        *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
                                       &found_dup_sk);
        if (*own_req) {
                tcp_move_syn(newtp, req);

                /* Clone pktoptions received with SYN, if we own the req */
                if (ireq->pktopts) {
                        newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk);
                        consume_skb(ireq->pktopts);
                        ireq->pktopts = NULL;
                        if (newnp->pktoptions)
                                tcp_v6_restore_cb(newnp->pktoptions);
                }
        } else {
                if (!req_unhash && found_dup_sk) {
                        /* This code path should only be executed in the
                         * syncookie case only
                         */
                        bh_unlock_sock(newsk);
                        sock_put(newsk);
                        newsk = NULL;
                }
        }

        return newsk;

exit_overflow:
        __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
exit_nonewsk:
        dst_release(dst);
exit:
        tcp_listendrop(sk);
        return NULL;
put_and_exit:
        inet_csk_prepare_forced_close(newsk);
        tcp_done(newsk);
        goto exit;
}

INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
                                                           u32));
/* The socket must have it's spinlock held when we get
 * here, unless it is a TCP_LISTEN socket.
 *
 * We have a potential double-lock case here, so even when
 * doing backlog processing we use the BH locking scheme.
 * This is because we cannot sleep with the original spinlock
 * held.
 */
INDIRECT_CALLABLE_SCOPE
int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
{
        struct ipv6_pinfo *np = tcp_inet6_sk(sk);
        struct sk_buff *opt_skb = NULL;
        enum skb_drop_reason reason;
        struct tcp_sock *tp;

        /* Imagine: socket is IPv6. IPv4 packet arrives,
           goes to IPv4 receive handler and backlogged.
           From backlog it always goes here. Kerboom...
           Fortunately, tcp_rcv_established and rcv_established
           handle them correctly, but it is not case with
           tcp_v6_hnd_req and tcp_v6_send_reset().   --ANK
         */

        if (skb->protocol == htons(ETH_P_IP))
                return tcp_v4_do_rcv(sk, skb);

        reason = psp_sk_rx_policy_check(sk, skb);
        if (reason)
                goto err_discard;

        /*
         *        socket locking is here for SMP purposes as backlog rcv
         *        is currently called with bh processing disabled.
         */

        /* Do Stevens' IPV6_PKTOPTIONS.

           Yes, guys, it is the only place in our code, where we
           may make it not affecting IPv4.
           The rest of code is protocol independent,
           and I do not like idea to uglify IPv4.

           Actually, all the idea behind IPV6_PKTOPTIONS
           looks not very well thought. For now we latch
           options, received in the last packet, enqueued
           by tcp. Feel free to propose better solution.
                                               --ANK (980728)
         */
        if (np->rxopt.all && sk->sk_state != TCP_LISTEN)
                opt_skb = skb_clone_and_charge_r(skb, sk);

        if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
                struct dst_entry *dst;

                dst = rcu_dereference_protected(sk->sk_rx_dst,
                                                lockdep_sock_is_held(sk));

                sock_rps_save_rxhash(sk, skb);
                sk_mark_napi_id(sk, skb);
                if (dst && unlikely(dst != skb_dst(skb))) {
                        if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
                            INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
                                            dst, sk->sk_rx_dst_cookie) == NULL) {
                                RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
                                dst_release(dst);
                        }
                }

                tcp_rcv_established(sk, skb);
                if (opt_skb)
                        goto ipv6_pktoptions;
                return 0;
        }

        if (tcp_checksum_complete(skb))
                goto csum_err;

        if (sk->sk_state == TCP_LISTEN) {
                struct sock *nsk = tcp_v6_cookie_check(sk, skb);

                if (!nsk)
                        return 0;
                if (nsk != sk) {
                        reason = tcp_child_process(sk, nsk, skb);
                        sock_put(nsk);
                        if (reason)
                                goto reset;
                        return 0;
                }
        } else
                sock_rps_save_rxhash(sk, skb);

        reason = tcp_rcv_state_process(sk, skb);
        if (reason)
                goto reset;
        if (opt_skb)
                goto ipv6_pktoptions;
        return 0;

reset:
        tcp_v6_send_reset(sk, skb, sk_rst_convert_drop_reason(reason));
discard:
        if (opt_skb)
                __kfree_skb(opt_skb);
        sk_skb_reason_drop(sk, skb, reason);
        return 0;
csum_err:
        reason = SKB_DROP_REASON_TCP_CSUM;
        trace_tcp_bad_csum(skb);
        TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
err_discard:
        TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
        goto discard;


ipv6_pktoptions:
        /* Do you ask, what is it?

           1. skb was enqueued by tcp.
           2. skb is added to tail of read queue, rather than out of order.
           3. socket is not in passive state.
           4. Finally, it really contains options, which user wants to receive.
         */
        tp = tcp_sk(sk);
        if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
            !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
                if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
                        WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb));
                if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
                        WRITE_ONCE(np->mcast_hops,
                                   ipv6_hdr(opt_skb)->hop_limit);
                if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
                        np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb));
                if (inet6_test_bit(REPFLOW, sk))
                        np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb));
                if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) {
                        tcp_v6_restore_cb(opt_skb);
                        opt_skb = xchg(&np->pktoptions, opt_skb);
                } else {
                        __kfree_skb(opt_skb);
                        opt_skb = xchg(&np->pktoptions, NULL);
                }
        }

        consume_skb(opt_skb);
        return 0;
}

static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
                           const struct tcphdr *th)
{
        /* This is tricky: we move IP6CB at its correct location into
         * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
         * _decode_session6() uses IP6CB().
         * barrier() makes sure compiler won't play aliasing games.
         */
        memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
                sizeof(struct inet6_skb_parm));
        barrier();

        TCP_SKB_CB(skb)->seq = ntohl(th->seq);
        TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
                                    skb->len - th->doff*4);
        TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
        TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th);
        TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
        TCP_SKB_CB(skb)->sacked = 0;
        TCP_SKB_CB(skb)->has_rxtstamp =
                        skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
}

INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
{
        struct net *net = dev_net_rcu(skb->dev);
        enum skb_drop_reason drop_reason;
        enum tcp_tw_status tw_status;
        int sdif = inet6_sdif(skb);
        int dif = inet6_iif(skb);
        const struct tcphdr *th;
        const struct ipv6hdr *hdr;
        struct sock *sk = NULL;
        bool refcounted;
        int ret;
        u32 isn;

        drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
        if (skb->pkt_type != PACKET_HOST)
                goto discard_it;

        /*
         *        Count it even if it's bad.
         */
        __TCP_INC_STATS(net, TCP_MIB_INSEGS);

        if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
                goto discard_it;

        th = (const struct tcphdr *)skb->data;

        if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
                drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
                goto bad_packet;
        }
        if (!pskb_may_pull(skb, th->doff*4))
                goto discard_it;

        if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
                goto csum_error;

        th = (const struct tcphdr *)skb->data;
        hdr = ipv6_hdr(skb);

lookup:
        sk = __inet6_lookup_skb(skb, __tcp_hdrlen(th),
                                th->source, th->dest, inet6_iif(skb), sdif,
                                &refcounted);
        if (!sk)
                goto no_tcp_socket;

        if (sk->sk_state == TCP_TIME_WAIT)
                goto do_time_wait;

        if (sk->sk_state == TCP_NEW_SYN_RECV) {
                struct request_sock *req = inet_reqsk(sk);
                bool req_stolen = false;
                struct sock *nsk;

                sk = req->rsk_listener;
                if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
                        drop_reason = SKB_DROP_REASON_XFRM_POLICY;
                else
                        drop_reason = tcp_inbound_hash(sk, req, skb,
                                                       &hdr->saddr, &hdr->daddr,
                                                       AF_INET6, dif, sdif);
                if (drop_reason) {
                        sk_drops_skbadd(sk, skb);
                        reqsk_put(req);
                        goto discard_it;
                }
                if (tcp_checksum_complete(skb)) {
                        reqsk_put(req);
                        goto csum_error;
                }
                if (unlikely(sk->sk_state != TCP_LISTEN)) {
                        nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
                        if (!nsk) {
                                inet_csk_reqsk_queue_drop_and_put(sk, req);
                                goto lookup;
                        }
                        sk = nsk;
                        /* reuseport_migrate_sock() has already held one sk_refcnt
                         * before returning.
                         */
                } else {
                        sock_hold(sk);
                }
                refcounted = true;
                nsk = NULL;
                drop_reason = tcp_filter(sk, skb);
                if (!drop_reason) {
                        th = (const struct tcphdr *)skb->data;
                        hdr = ipv6_hdr(skb);
                        tcp_v6_fill_cb(skb, hdr, th);
                        nsk = tcp_check_req(sk, skb, req, false, &req_stolen,
                                            &drop_reason);
                }
                if (!nsk) {
                        reqsk_put(req);
                        if (req_stolen) {
                                /* Another cpu got exclusive access to req
                                 * and created a full blown socket.
                                 * Try to feed this packet to this socket
                                 * instead of discarding it.
                                 */
                                tcp_v6_restore_cb(skb);
                                sock_put(sk);
                                goto lookup;
                        }
                        goto discard_and_relse;
                }
                nf_reset_ct(skb);
                if (nsk == sk) {
                        reqsk_put(req);
                        tcp_v6_restore_cb(skb);
                } else {
                        drop_reason = tcp_child_process(sk, nsk, skb);
                        if (drop_reason) {
                                enum sk_rst_reason rst_reason;

                                rst_reason = sk_rst_convert_drop_reason(drop_reason);
                                tcp_v6_send_reset(nsk, skb, rst_reason);
                                sock_put(nsk);
                                goto discard_and_relse;
                        }
                        sock_put(nsk);
                        sock_put(sk);
                        return 0;
                }
        }

        isn = 0;
process:
        if (static_branch_unlikely(&ip6_min_hopcount)) {
                /* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
                if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) {
                        __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
                        drop_reason = SKB_DROP_REASON_TCP_MINTTL;
                        goto discard_and_relse;
                }
        }

        if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
                drop_reason = SKB_DROP_REASON_XFRM_POLICY;
                goto discard_and_relse;
        }

        drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr,
                                       AF_INET6, dif, sdif);
        if (drop_reason)
                goto discard_and_relse;

        nf_reset_ct(skb);

        drop_reason = tcp_filter(sk, skb);
        if (drop_reason)
                goto discard_and_relse;

        th = (const struct tcphdr *)skb->data;
        hdr = ipv6_hdr(skb);
        tcp_v6_fill_cb(skb, hdr, th);
        TCP_SKB_CB(skb)->tcp_tw_isn = isn;

        skb->dev = NULL;

        if (sk->sk_state == TCP_LISTEN) {
                ret = tcp_v6_do_rcv(sk, skb);
                goto put_and_return;
        }

        sk_incoming_cpu_update(sk);

        bh_lock_sock_nested(sk);
        tcp_segs_in(tcp_sk(sk), skb);
        ret = 0;
        if (!sock_owned_by_user(sk)) {
                ret = tcp_v6_do_rcv(sk, skb);
        } else {
                drop_reason = tcp_add_backlog(sk, skb);
                if (drop_reason)
                        goto discard_and_relse;
        }
        bh_unlock_sock(sk);
put_and_return:
        if (refcounted)
                sock_put(sk);
        return ret ? -1 : 0;

no_tcp_socket:
        drop_reason = SKB_DROP_REASON_NO_SOCKET;
        if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
                goto discard_it;

        tcp_v6_fill_cb(skb, hdr, th);

        if (tcp_checksum_complete(skb)) {
csum_error:
                drop_reason = SKB_DROP_REASON_TCP_CSUM;
                trace_tcp_bad_csum(skb);
                __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
bad_packet:
                __TCP_INC_STATS(net, TCP_MIB_INERRS);
        } else {
                tcp_v6_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason));
        }

discard_it:
        SKB_DR_OR(drop_reason, NOT_SPECIFIED);
        sk_skb_reason_drop(sk, skb, drop_reason);
        return 0;

discard_and_relse:
        sk_drops_skbadd(sk, skb);
        if (refcounted)
                sock_put(sk);
        goto discard_it;

do_time_wait:
        if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) {
                drop_reason = SKB_DROP_REASON_XFRM_POLICY;
                inet_twsk_put(inet_twsk(sk));
                goto discard_it;
        }

        tcp_v6_fill_cb(skb, hdr, th);

        if (tcp_checksum_complete(skb)) {
                inet_twsk_put(inet_twsk(sk));
                goto csum_error;
        }

        tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn,
                                               &drop_reason);
        switch (tw_status) {
        case TCP_TW_SYN:
        {
                struct sock *sk2;

                sk2 = inet6_lookup_listener(net, skb, __tcp_hdrlen(th),
                                            &ipv6_hdr(skb)->saddr, th->source,
                                            &ipv6_hdr(skb)->daddr,
                                            ntohs(th->dest),
                                            tcp_v6_iif_l3_slave(skb),
                                            sdif);
                if (sk2) {
                        struct inet_timewait_sock *tw = inet_twsk(sk);
                        inet_twsk_deschedule_put(tw);
                        sk = sk2;
                        tcp_v6_restore_cb(skb);
                        refcounted = false;
                        goto process;
                }

                drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb);
                if (drop_reason) {
                        inet_twsk_put(inet_twsk(sk));
                        goto discard_it;
                }
        }
                /* to ACK */
                fallthrough;
        case TCP_TW_ACK:
        case TCP_TW_ACK_OOW:
                tcp_v6_timewait_ack(sk, skb, tw_status);
                break;
        case TCP_TW_RST:
                tcp_v6_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
                inet_twsk_deschedule_put(inet_twsk(sk));
                goto discard_it;
        case TCP_TW_SUCCESS:
                ;
        }
        goto discard_it;
}

static struct timewait_sock_ops tcp6_timewait_sock_ops = {
        .twsk_obj_size        = sizeof(struct tcp6_timewait_sock),
};

const struct inet_connection_sock_af_ops ipv6_specific = {
        .queue_xmit           = inet6_csk_xmit,
        .rebuild_header           = inet6_sk_rebuild_header,
        .sk_rx_dst_set           = inet6_sk_rx_dst_set,
        .conn_request           = tcp_v6_conn_request,
        .syn_recv_sock           = tcp_v6_syn_recv_sock,
        .net_header_len           = sizeof(struct ipv6hdr),
        .setsockopt           = ipv6_setsockopt,
        .getsockopt           = ipv6_getsockopt,
        .mtu_reduced           = tcp_v6_mtu_reduced,
};

#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
#ifdef CONFIG_TCP_MD5SIG
        .md5_lookup        =        tcp_v6_md5_lookup,
        .calc_md5_hash        =        tcp_v6_md5_hash_skb,
        .md5_parse        =        tcp_v6_parse_md5_keys,
#endif
#ifdef CONFIG_TCP_AO
        .ao_lookup        =        tcp_v6_ao_lookup,
        .calc_ao_hash        =        tcp_v6_ao_hash_skb,
        .ao_parse        =        tcp_v6_parse_ao,
        .ao_calc_key_sk        =        tcp_v6_ao_calc_key_sk,
#endif
};
#endif

/*
 *        TCP over IPv4 via INET6 API
 */
static const struct inet_connection_sock_af_ops ipv6_mapped = {
        .queue_xmit           = ip_queue_xmit,
        .rebuild_header           = inet_sk_rebuild_header,
        .sk_rx_dst_set           = inet_sk_rx_dst_set,
        .conn_request           = tcp_v6_conn_request,
        .syn_recv_sock           = tcp_v6_syn_recv_sock,
        .net_header_len           = sizeof(struct iphdr),
        .setsockopt           = ipv6_setsockopt,
        .getsockopt           = ipv6_getsockopt,
        .mtu_reduced           = tcp_v4_mtu_reduced,
};

#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
#ifdef CONFIG_TCP_MD5SIG
        .md5_lookup        =        tcp_v4_md5_lookup,
        .calc_md5_hash        =        tcp_v4_md5_hash_skb,
        .md5_parse        =        tcp_v6_parse_md5_keys,
#endif
#ifdef CONFIG_TCP_AO
        .ao_lookup        =        tcp_v6_ao_lookup,
        .calc_ao_hash        =        tcp_v4_ao_hash_skb,
        .ao_parse        =        tcp_v6_parse_ao,
        .ao_calc_key_sk        =        tcp_v4_ao_calc_key_sk,
#endif
};

static void tcp6_destruct_sock(struct sock *sk)
{
        tcp_md5_destruct_sock(sk);
        tcp_ao_destroy_sock(sk, false);
        inet6_sock_destruct(sk);
}
#endif

/* NOTE: A lot of things set to zero explicitly by call to
 *       sk_alloc() so need not be done here.
 */
static int tcp_v6_init_sock(struct sock *sk)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        tcp_init_sock(sk);

        icsk->icsk_af_ops = &ipv6_specific;

#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
        tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
        sk->sk_destruct = tcp6_destruct_sock;
#endif

        return 0;
}

#ifdef CONFIG_PROC_FS
/* Proc filesystem TCPv6 sock list dumping. */
static void get_openreq6(struct seq_file *seq,
                         const struct request_sock *req, int i)
{
        long ttd = req->rsk_timer.expires - jiffies;
        const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr;
        const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr;

        if (ttd < 0)
                ttd = 0;

        seq_printf(seq,
                   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
                   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d 0\n",
                   i,
                   src->s6_addr32[0], src->s6_addr32[1],
                   src->s6_addr32[2], src->s6_addr32[3],
                   inet_rsk(req)->ir_num,
                   dest->s6_addr32[0], dest->s6_addr32[1],
                   dest->s6_addr32[2], dest->s6_addr32[3],
                   ntohs(inet_rsk(req)->ir_rmt_port),
                   TCP_SYN_RECV,
                   0, 0, /* could print option size, but that is af dependent. */
                   1,   /* timers active (only the expire timer) */
                   jiffies_to_clock_t(ttd),
                   req->num_timeout,
                   from_kuid_munged(seq_user_ns(seq),
                                    sk_uid(req->rsk_listener)),
                   0,  /* non standard timer */
                   0, /* open_requests have no inode */
                   0);
}

static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
{
        const struct in6_addr *dest, *src;
        __u16 destp, srcp;
        int timer_active;
        unsigned long timer_expires;
        const struct inet_sock *inet = inet_sk(sp);
        const struct tcp_sock *tp = tcp_sk(sp);
        const struct inet_connection_sock *icsk = inet_csk(sp);
        const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
        u8 icsk_pending;
        int rx_queue;
        int state;

        dest  = &sp->sk_v6_daddr;
        src   = &sp->sk_v6_rcv_saddr;
        destp = ntohs(inet->inet_dport);
        srcp  = ntohs(inet->inet_sport);

        icsk_pending = smp_load_acquire(&icsk->icsk_pending);
        if (icsk_pending == ICSK_TIME_RETRANS ||
            icsk_pending == ICSK_TIME_REO_TIMEOUT ||
            icsk_pending == ICSK_TIME_LOSS_PROBE) {
                timer_active        = 1;
                timer_expires        = tcp_timeout_expires(sp);
        } else if (icsk_pending == ICSK_TIME_PROBE0) {
                timer_active        = 4;
                timer_expires        = tcp_timeout_expires(sp);
        } else if (timer_pending(&icsk->icsk_keepalive_timer)) {
                timer_active        = 2;
                timer_expires        = icsk->icsk_keepalive_timer.expires;
        } else {
                timer_active        = 0;
                timer_expires = jiffies;
        }

        state = inet_sk_state_load(sp);
        if (state == TCP_LISTEN)
                rx_queue = READ_ONCE(sp->sk_ack_backlog);
        else
                /* Because we don't lock the socket,
                 * we might find a transient negative value.
                 */
                rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
                                      READ_ONCE(tp->copied_seq), 0);

        seq_printf(seq,
                   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
                   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d 0 %lu %lu %u %u %d\n",
                   i,
                   src->s6_addr32[0], src->s6_addr32[1],
                   src->s6_addr32[2], src->s6_addr32[3], srcp,
                   dest->s6_addr32[0], dest->s6_addr32[1],
                   dest->s6_addr32[2], dest->s6_addr32[3], destp,
                   state,
                   READ_ONCE(tp->write_seq) - tp->snd_una,
                   rx_queue,
                   timer_active,
                   jiffies_delta_to_clock_t(timer_expires - jiffies),
                   READ_ONCE(icsk->icsk_retransmits),
                   from_kuid_munged(seq_user_ns(seq), sk_uid(sp)),
                   READ_ONCE(icsk->icsk_probes_out),
                   sock_i_ino(sp),
                   refcount_read(&sp->sk_refcnt),
                   jiffies_to_clock_t(icsk->icsk_rto),
                   jiffies_to_clock_t(icsk->icsk_ack.ato),
                   (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
                   tcp_snd_cwnd(tp),
                   state == TCP_LISTEN ?
                        fastopenq->max_qlen :
                        (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
                   );
}

static void get_timewait6_sock(struct seq_file *seq,
                               struct inet_timewait_sock *tw, int i)
{
        long delta = tw->tw_timer.expires - jiffies;
        const struct in6_addr *dest, *src;
        __u16 destp, srcp;

        dest = &tw->tw_v6_daddr;
        src  = &tw->tw_v6_rcv_saddr;
        destp = ntohs(tw->tw_dport);
        srcp  = ntohs(tw->tw_sport);

        seq_printf(seq,
                   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
                   "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n",
                   i,
                   src->s6_addr32[0], src->s6_addr32[1],
                   src->s6_addr32[2], src->s6_addr32[3], srcp,
                   dest->s6_addr32[0], dest->s6_addr32[1],
                   dest->s6_addr32[2], dest->s6_addr32[3], destp,
                   READ_ONCE(tw->tw_substate), 0, 0,
                   3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
                   refcount_read(&tw->tw_refcnt));
}

static int tcp6_seq_show(struct seq_file *seq, void *v)
{
        struct tcp_iter_state *st;
        struct sock *sk = v;

        if (v == SEQ_START_TOKEN) {
                seq_puts(seq,
                         "  sl  "
                         "local_address                         "
                         "remote_address                        "
                         "st tx_queue rx_queue tr tm->when retrnsmt"
                         "   uid  timeout inode\n");
                goto out;
        }
        st = seq->private;

        if (sk->sk_state == TCP_TIME_WAIT)
                get_timewait6_sock(seq, v, st->num);
        else if (sk->sk_state == TCP_NEW_SYN_RECV)
                get_openreq6(seq, v, st->num);
        else
                get_tcp6_sock(seq, v, st->num);
out:
        return 0;
}

static const struct seq_operations tcp6_seq_ops = {
        .show                = tcp6_seq_show,
        .start                = tcp_seq_start,
        .next                = tcp_seq_next,
        .stop                = tcp_seq_stop,
};

static struct tcp_seq_afinfo tcp6_seq_afinfo = {
        .family                = AF_INET6,
};

int __net_init tcp6_proc_init(struct net *net)
{
        if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops,
                        sizeof(struct tcp_iter_state), &tcp6_seq_afinfo))
                return -ENOMEM;
        return 0;
}

void tcp6_proc_exit(struct net *net)
{
        remove_proc_entry("tcp6", net->proc_net);
}
#endif

struct proto tcpv6_prot = {
        .name                        = "TCPv6",
        .owner                        = THIS_MODULE,
        .close                        = tcp_close,
        .pre_connect                = tcp_v6_pre_connect,
        .connect                = tcp_v6_connect,
        .disconnect                = tcp_disconnect,
        .accept                        = inet_csk_accept,
        .ioctl                        = tcp_ioctl,
        .init                        = tcp_v6_init_sock,
        .destroy                = tcp_v4_destroy_sock,
        .shutdown                = tcp_shutdown,
        .setsockopt                = tcp_setsockopt,
        .getsockopt                = tcp_getsockopt,
        .bpf_bypass_getsockopt        = tcp_bpf_bypass_getsockopt,
        .keepalive                = tcp_set_keepalive,
        .recvmsg                = tcp_recvmsg,
        .sendmsg                = tcp_sendmsg,
        .splice_eof                = tcp_splice_eof,
        .backlog_rcv                = tcp_v6_do_rcv,
        .release_cb                = tcp_release_cb,
        .hash                        = inet_hash,
        .unhash                        = inet_unhash,
        .get_port                = inet_csk_get_port,
        .put_port                = inet_put_port,
#ifdef CONFIG_BPF_SYSCALL
        .psock_update_sk_prot        = tcp_bpf_update_proto,
#endif
        .enter_memory_pressure        = tcp_enter_memory_pressure,
        .leave_memory_pressure        = tcp_leave_memory_pressure,
        .stream_memory_free        = tcp_stream_memory_free,
        .sockets_allocated        = &tcp_sockets_allocated,

        .memory_allocated        = &net_aligned_data.tcp_memory_allocated,
        .per_cpu_fw_alloc        = &tcp_memory_per_cpu_fw_alloc,

        .memory_pressure        = &tcp_memory_pressure,
        .sysctl_mem                = sysctl_tcp_mem,
        .sysctl_wmem_offset        = offsetof(struct net, ipv4.sysctl_tcp_wmem),
        .sysctl_rmem_offset        = offsetof(struct net, ipv4.sysctl_tcp_rmem),
        .max_header                = MAX_TCP_HEADER,
        .obj_size                = sizeof(struct tcp6_sock),
        .freeptr_offset                = offsetof(struct tcp6_sock,
                                           tcp.inet_conn.icsk_inet.sk.sk_freeptr),
        .ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6),
        .slab_flags                = SLAB_TYPESAFE_BY_RCU,
        .twsk_prot                = &tcp6_timewait_sock_ops,
        .rsk_prot                = &tcp6_request_sock_ops,
        .h.hashinfo                = NULL,
        .no_autobind                = true,
        .diag_destroy                = tcp_abort,
};
EXPORT_SYMBOL_GPL(tcpv6_prot);


static struct inet_protosw tcpv6_protosw = {
        .type                =        SOCK_STREAM,
        .protocol        =        IPPROTO_TCP,
        .prot                =        &tcpv6_prot,
        .ops                =        &inet6_stream_ops,
        .flags                =        INET_PROTOSW_PERMANENT |
                                INET_PROTOSW_ICSK,
};

static int __net_init tcpv6_net_init(struct net *net)
{
        int res;

        res = inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6,
                                   SOCK_RAW, IPPROTO_TCP, net);
        if (!res)
                net->ipv6.tcp_sk->sk_clockid = CLOCK_MONOTONIC;

        return res;
}

static void __net_exit tcpv6_net_exit(struct net *net)
{
        inet_ctl_sock_destroy(net->ipv6.tcp_sk);
}

static struct pernet_operations tcpv6_net_ops = {
        .init            = tcpv6_net_init,
        .exit            = tcpv6_net_exit,
};

int __init tcpv6_init(void)
{
        int ret;

        net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
                .handler     = tcp_v6_rcv,
                .err_handler = tcp_v6_err,
                .flags             = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
        };
        ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
        if (ret)
                goto out;

        /* register inet6 protocol */
        ret = inet6_register_protosw(&tcpv6_protosw);
        if (ret)
                goto out_tcpv6_protocol;

        ret = register_pernet_subsys(&tcpv6_net_ops);
        if (ret)
                goto out_tcpv6_protosw;

        ret = mptcpv6_init();
        if (ret)
                goto out_tcpv6_pernet_subsys;

out:
        return ret;

out_tcpv6_pernet_subsys:
        unregister_pernet_subsys(&tcpv6_net_ops);
out_tcpv6_protosw:
        inet6_unregister_protosw(&tcpv6_protosw);
out_tcpv6_protocol:
        inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
        goto out;
}

void tcpv6_exit(void)
{
        unregister_pernet_subsys(&tcpv6_net_ops);
        inet6_unregister_protosw(&tcpv6_protosw);
        inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
}




































































































    1 





    1 












































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2003-2011 Jozsef Kadlecsik <kadlec@netfilter.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

/* Get Layer-4 data from the packets */

#include <linux/ip.h>
#include <linux/skbuff.h>
#include <linux/icmp.h>
#include <linux/icmpv6.h>
#include <linux/sctp.h>
#include <linux/netfilter_ipv6/ip6_tables.h>
#include <net/ip.h>
#include <net/ipv6.h>

#include <linux/netfilter/ipset/ip_set_getport.h>
#include <linux/export.h>

/* We must handle non-linear skbs */
static bool
get_port(const struct sk_buff *skb, int protocol, unsigned int protooff,
         bool src, __be16 *port, u8 *proto)
{
        switch (protocol) {
        case IPPROTO_TCP: {
                struct tcphdr _tcph;
                const struct tcphdr *th;

                th = skb_header_pointer(skb, protooff, sizeof(_tcph), &_tcph);
                if (!th)
                        /* No choice either */
                        return false;

                *port = src ? th->source : th->dest;
                break;
        }
        case IPPROTO_SCTP: {
                struct sctphdr _sh;
                const struct sctphdr *sh;

                sh = skb_header_pointer(skb, protooff, sizeof(_sh), &_sh);
                if (!sh)
                        /* No choice either */
                        return false;

                *port = src ? sh->source : sh->dest;
                break;
        }
        case IPPROTO_UDP:
        case IPPROTO_UDPLITE: {
                struct udphdr _udph;
                const struct udphdr *uh;

                uh = skb_header_pointer(skb, protooff, sizeof(_udph), &_udph);
                if (!uh)
                        /* No choice either */
                        return false;

                *port = src ? uh->source : uh->dest;
                break;
        }
        case IPPROTO_ICMP: {
                struct icmphdr _ich;
                const struct icmphdr *ic;

                ic = skb_header_pointer(skb, protooff, sizeof(_ich), &_ich);
                if (!ic)
                        return false;

                *port = (__force __be16)htons((ic->type << 8) | ic->code);
                break;
        }
        case IPPROTO_ICMPV6: {
                struct icmp6hdr _ich;
                const struct icmp6hdr *ic;

                ic = skb_header_pointer(skb, protooff, sizeof(_ich), &_ich);
                if (!ic)
                        return false;

                *port = (__force __be16)
                        htons((ic->icmp6_type << 8) | ic->icmp6_code);
                break;
        }
        default:
                break;
        }
        *proto = protocol;

        return true;
}

bool
ip_set_get_ip4_port(const struct sk_buff *skb, bool src,
                    __be16 *port, u8 *proto)
{
        const struct iphdr *iph = ip_hdr(skb);
        unsigned int protooff = skb_network_offset(skb) + ip_hdrlen(skb);
        int protocol = iph->protocol;

        /* See comments at tcp_match in ip_tables.c */
        if (protocol <= 0)
                return false;

        if (ntohs(iph->frag_off) & IP_OFFSET)
                switch (protocol) {
                case IPPROTO_TCP:
                case IPPROTO_SCTP:
                case IPPROTO_UDP:
                case IPPROTO_UDPLITE:
                case IPPROTO_ICMP:
                        /* Port info not available for fragment offset > 0 */
                        return false;
                default:
                        /* Other protocols doesn't have ports,
                         * so we can match fragments.
                         */
                        *proto = protocol;
                        return true;
                }

        return get_port(skb, protocol, protooff, src, port, proto);
}
EXPORT_SYMBOL_GPL(ip_set_get_ip4_port);

#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES)
bool
ip_set_get_ip6_port(const struct sk_buff *skb, bool src,
                    __be16 *port, u8 *proto)
{
        int protoff;
        u8 nexthdr;
        __be16 frag_off = 0;

        nexthdr = ipv6_hdr(skb)->nexthdr;
        protoff = ipv6_skip_exthdr(skb,
                                   skb_network_offset(skb) +
                                        sizeof(struct ipv6hdr), &nexthdr,
                                   &frag_off);
        if (protoff < 0 || (frag_off & htons(~0x7)) != 0)
                return false;

        return get_port(skb, nexthdr, protoff, src, port, proto);
}
EXPORT_SYMBOL_GPL(ip_set_get_ip6_port);
#endif






























































































































    1 










































































































    2 






























































































































































































































    2 


    2 


    2 


































    2 



    2 
    2 

    2 












    2 








































































































































    2 





























    2 
    2 


    2 















   29 









































































































































































    2 









    2 





















    2 










    2 




















































































































































































































































































    1 

    1 

























    1 







    1 































































































































































































    2 


    2 















    2 











    2 



    2 


















    2 












    2 
    2 







    2 
    2 





    2 








    2 


    2 









    2 


    2 


    2 




    2 








    2 
    2 








































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * zswap.c - zswap driver file
 *
 * zswap is a cache that takes pages that are in the process
 * of being swapped out and attempts to compress and store them in a
 * RAM-based memory pool.  This can result in a significant I/O reduction on
 * the swap device and, in the case where decompressing from RAM is faster
 * than reading from the swap device, can also improve workload performance.
 *
 * Copyright (C) 2012  Seth Jennings <sjenning@linux.vnet.ibm.com>
*/

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/cpu.h>
#include <linux/highmem.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/atomic.h>
#include <linux/swap_ops.h>
#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <linux/mempolicy.h>
#include <linux/mempool.h>
#include <crypto/acompress.h>
#include <crypto/scatterwalk.h>
#include <linux/zswap.h>
#include <linux/mm_types.h>
#include <linux/page-flags.h>
#include <linux/swapops.h>
#include <linux/writeback.h>
#include <linux/pagemap.h>
#include <linux/workqueue.h>
#include <linux/list_lru.h>
#include <linux/zsmalloc.h>

#include "swap.h"
#include "internal.h"

/*********************************
* statistics
**********************************/
/* The number of pages currently stored in zswap */
atomic_long_t zswap_stored_pages = ATOMIC_LONG_INIT(0);
/* The number of incompressible pages currently stored in zswap */
static atomic_long_t zswap_stored_incompressible_pages = ATOMIC_LONG_INIT(0);

/*
 * The statistics below are not protected from concurrent access for
 * performance reasons so they may not be a 100% accurate.  However,
 * they do provide useful information on roughly how many times a
 * certain event is occurring.
*/

/* Pool limit was hit (see zswap_max_pool_percent) */
static u64 zswap_pool_limit_hit;
/* Pages written back when pool limit was reached */
static u64 zswap_written_back_pages;
/* Store failed due to a reclaim failure after pool limit was reached */
static u64 zswap_reject_reclaim_fail;
/* Store failed due to compression algorithm failure */
static u64 zswap_reject_compress_fail;
/* Compressed page was too big for the allocator to (optimally) store */
static u64 zswap_reject_compress_poor;
/* Load or writeback failed due to decompression failure */
static u64 zswap_decompress_fail;
/* Store failed because underlying allocator could not get memory */
static u64 zswap_reject_alloc_fail;
/* Store failed because the entry metadata could not be allocated (rare) */
static u64 zswap_reject_kmemcache_fail;

/* Shrinker work queue */
static struct workqueue_struct *shrink_wq;
/* Pool limit was hit, we need to calm down */
static bool zswap_pool_reached_full;

/*********************************
* tunables
**********************************/

#define ZSWAP_PARAM_UNSET ""

static int zswap_setup(void);

/* Enable/disable zswap */
static DEFINE_STATIC_KEY_MAYBE(CONFIG_ZSWAP_DEFAULT_ON, zswap_ever_enabled);
static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON);
static int zswap_enabled_param_set(const char *,
                                   const struct kernel_param *);
static const struct kernel_param_ops zswap_enabled_param_ops = {
        .set =                zswap_enabled_param_set,
        .get =                param_get_bool,
};
module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644);

/* Crypto compressor to use */
static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
static int zswap_compressor_param_set(const char *,
                                      const struct kernel_param *);
static const struct kernel_param_ops zswap_compressor_param_ops = {
        .set =                zswap_compressor_param_set,
        .get =                param_get_charp,
        .free =                param_free_charp,
};
module_param_cb(compressor, &zswap_compressor_param_ops,
                &zswap_compressor, 0644);

/* The maximum percentage of memory that the compressed pool can occupy */
static unsigned int zswap_max_pool_percent = 20;
module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);

/* The threshold for accepting new pages after the max_pool_percent was hit */
static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */
module_param_named(accept_threshold_percent, zswap_accept_thr_percent,
                   uint, 0644);

/* Enable/disable memory pressure-based shrinker. */
static bool zswap_shrinker_enabled = IS_ENABLED(
                CONFIG_ZSWAP_SHRINKER_DEFAULT_ON);
module_param_named(shrinker_enabled, zswap_shrinker_enabled, bool, 0644);

bool zswap_is_enabled(void)
{
        return zswap_enabled;
}

bool zswap_never_enabled(void)
{
        return !static_branch_maybe(CONFIG_ZSWAP_DEFAULT_ON, &zswap_ever_enabled);
}

/*********************************
* data structures
**********************************/

struct crypto_acomp_ctx {
        struct crypto_acomp *acomp;
        struct acomp_req *req;
        struct crypto_wait wait;
        u8 *buffer;
        struct mutex mutex;
};

/*
 * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock.
 * The only case where lru_lock is not acquired while holding tree.lock is
 * when a zswap_entry is taken off the lru for writeback, in that case it
 * needs to be verified that it's still valid in the tree.
 */
struct zswap_pool {
        struct zs_pool *zs_pool;
        struct crypto_acomp_ctx __percpu *acomp_ctx;
        struct percpu_ref ref;
        struct list_head list;
        struct work_struct release_work;
        struct hlist_node node;
        char tfm_name[CRYPTO_MAX_ALG_NAME];
};

/* Global LRU lists shared by all zswap pools. */
static struct list_lru zswap_list_lru;

/* The lock protects zswap_next_shrink updates. */
static DEFINE_SPINLOCK(zswap_shrink_lock);
static struct mem_cgroup *zswap_next_shrink;
static struct work_struct zswap_shrink_work;
static struct shrinker *zswap_shrinker;

/*
 * struct zswap_entry
 *
 * This structure contains the metadata for tracking a single compressed
 * page within zswap.
 *
 * swpentry - associated swap entry, the offset indexes into the xarray
 * length - the length in bytes of the compressed page data.  Needed during
 *          decompression.
 * referenced - true if the entry recently entered the zswap pool. Unset by the
 *              writeback logic. The entry is only reclaimed by the writeback
 *              logic if referenced is unset. See comments in the shrinker
 *              section for context.
 * pool - the zswap_pool the entry's data is in
 * handle - zsmalloc allocation handle that stores the compressed page data
 * objcg - the obj_cgroup that the compressed memory is charged to
 * lru - handle to the pool's lru used to evict pages.
 */
struct zswap_entry {
        swp_entry_t swpentry;
        unsigned int length;
        bool referenced;
        struct zswap_pool *pool;
        unsigned long handle;
        struct obj_cgroup *objcg;
        struct list_head lru;
};

static struct xarray *zswap_trees[MAX_SWAPFILES];
static unsigned int nr_zswap_trees[MAX_SWAPFILES];

/* RCU-protected iteration */
static LIST_HEAD(zswap_pools);
/* protects zswap_pools list modification */
static DEFINE_SPINLOCK(zswap_pools_lock);
/* pool counter to provide unique names to zsmalloc */
static atomic_t zswap_pools_count = ATOMIC_INIT(0);

enum zswap_init_type {
        ZSWAP_UNINIT,
        ZSWAP_INIT_SUCCEED,
        ZSWAP_INIT_FAILED
};

static enum zswap_init_type zswap_init_state;

/* used to ensure the integrity of initialization */
static DEFINE_MUTEX(zswap_init_lock);

/* init completed, but couldn't create the initial pool */
static bool zswap_has_pool;

/*********************************
* helpers and fwd declarations
**********************************/

/* One swap address space for each 64M swap space */
#define ZSWAP_ADDRESS_SPACE_SHIFT 14
#define ZSWAP_ADDRESS_SPACE_PAGES (1 << ZSWAP_ADDRESS_SPACE_SHIFT)
static inline struct xarray *swap_zswap_tree(swp_entry_t swp)
{
        return &zswap_trees[swp_type(swp)][swp_offset(swp)
                >> ZSWAP_ADDRESS_SPACE_SHIFT];
}

#define zswap_pool_debug(msg, p)                        \
        pr_debug("%s pool %s\n", msg, (p)->tfm_name)

/*********************************
* pool functions
**********************************/
static void __zswap_pool_empty(struct percpu_ref *ref);

static void acomp_ctx_free(struct crypto_acomp_ctx *acomp_ctx)
{
        if (!acomp_ctx)
                return;

        /*
         * If there was an error in allocating @acomp_ctx->req, it
         * would be set to NULL.
         */
        if (acomp_ctx->req)
                acomp_request_free(acomp_ctx->req);

        acomp_ctx->req = NULL;

        /*
         * We have to handle both cases here: an error pointer return from
         * crypto_alloc_acomp_node(); and a) NULL initialization by zswap, or
         * b) NULL assignment done in a previous call to acomp_ctx_free().
         */
        if (!IS_ERR_OR_NULL(acomp_ctx->acomp))
                crypto_free_acomp(acomp_ctx->acomp);

        acomp_ctx->acomp = NULL;

        kfree(acomp_ctx->buffer);
        acomp_ctx->buffer = NULL;
}

static struct zswap_pool *zswap_pool_create(char *compressor)
{
        struct zswap_pool *pool;
        char name[38]; /* 'zswap' + 32 char (max) num + \0 */
        int ret, cpu;

        if (!zswap_has_pool && !strcmp(compressor, ZSWAP_PARAM_UNSET))
                return NULL;

        pool = kzalloc_obj(*pool);
        if (!pool)
                return NULL;

        /* unique name for each pool specifically required by zsmalloc */
        snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
        pool->zs_pool = zs_create_pool(name);
        if (!pool->zs_pool)
                goto error;

        strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));

        /* Many things rely on the zero-initialization. */
        pool->acomp_ctx = alloc_percpu_gfp(*pool->acomp_ctx,
                                           GFP_KERNEL | __GFP_ZERO);
        if (!pool->acomp_ctx) {
                pr_err("percpu alloc failed\n");
                goto error;
        }

        /*
         * This is serialized against CPU hotplug operations. Hence, cores
         * cannot be offlined until this finishes.
         */
        ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
                                       &pool->node);

        /*
         * cpuhp_state_add_instance() will not cleanup on failure since
         * we don't register a hotunplug callback.
         */
        if (ret)
                goto cpuhp_add_fail;

        /* being the current pool takes 1 ref; this func expects the
         * caller to always add the new pool as the current pool
         */
        ret = percpu_ref_init(&pool->ref, __zswap_pool_empty,
                              PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
        if (ret)
                goto ref_fail;
        INIT_LIST_HEAD(&pool->list);

        zswap_pool_debug("created", pool);

        return pool;

ref_fail:
        cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);

cpuhp_add_fail:
        for_each_possible_cpu(cpu)
                acomp_ctx_free(per_cpu_ptr(pool->acomp_ctx, cpu));
error:
        if (pool->acomp_ctx)
                free_percpu(pool->acomp_ctx);
        if (pool->zs_pool)
                zs_destroy_pool(pool->zs_pool);
        kfree(pool);
        return NULL;
}

static struct zswap_pool *__zswap_pool_create_fallback(void)
{
        if (!crypto_has_acomp(zswap_compressor, 0, 0) &&
            strcmp(zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) {
                pr_err("compressor %s not available, using default %s\n",
                       zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT);
                param_free_charp(&zswap_compressor);
                zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
        }

        /* Default compressor should be available. Kconfig bug? */
        if (WARN_ON_ONCE(!crypto_has_acomp(zswap_compressor, 0, 0))) {
                zswap_compressor = ZSWAP_PARAM_UNSET;
                return NULL;
        }

        return zswap_pool_create(zswap_compressor);
}

static void zswap_pool_destroy(struct zswap_pool *pool)
{
        int cpu;

        zswap_pool_debug("destroying", pool);

        cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);

        for_each_possible_cpu(cpu)
                acomp_ctx_free(per_cpu_ptr(pool->acomp_ctx, cpu));

        free_percpu(pool->acomp_ctx);

        zs_destroy_pool(pool->zs_pool);
        kfree(pool);
}

static void __zswap_pool_release(struct work_struct *work)
{
        struct zswap_pool *pool = container_of(work, typeof(*pool),
                                                release_work);

        synchronize_rcu();

        /* nobody should have been able to get a ref... */
        WARN_ON(!percpu_ref_is_zero(&pool->ref));
        percpu_ref_exit(&pool->ref);

        /* pool is now off zswap_pools list and has no references. */
        zswap_pool_destroy(pool);
}

static struct zswap_pool *zswap_pool_current(void);

static void __zswap_pool_empty(struct percpu_ref *ref)
{
        struct zswap_pool *pool;

        pool = container_of(ref, typeof(*pool), ref);

        spin_lock_bh(&zswap_pools_lock);

        WARN_ON(pool == zswap_pool_current());

        list_del_rcu(&pool->list);

        INIT_WORK(&pool->release_work, __zswap_pool_release);
        schedule_work(&pool->release_work);

        spin_unlock_bh(&zswap_pools_lock);
}

static int __must_check zswap_pool_tryget(struct zswap_pool *pool)
{
        if (!pool)
                return 0;

        return percpu_ref_tryget(&pool->ref);
}

/* The caller must already have a reference. */
static void zswap_pool_get(struct zswap_pool *pool)
{
        percpu_ref_get(&pool->ref);
}

static void zswap_pool_put(struct zswap_pool *pool)
{
        percpu_ref_put(&pool->ref);
}

static struct zswap_pool *__zswap_pool_current(void)
{
        struct zswap_pool *pool;

        pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
        WARN_ONCE(!pool && zswap_has_pool,
                  "%s: no page storage pool!\n", __func__);

        return pool;
}

static struct zswap_pool *zswap_pool_current(void)
{
        assert_spin_locked(&zswap_pools_lock);

        return __zswap_pool_current();
}

static struct zswap_pool *zswap_pool_current_get(void)
{
        struct zswap_pool *pool;

        rcu_read_lock();

        pool = __zswap_pool_current();
        if (!zswap_pool_tryget(pool))
                pool = NULL;

        rcu_read_unlock();

        return pool;
}

/* type and compressor must be null-terminated */
static struct zswap_pool *zswap_pool_find_get(char *compressor)
{
        struct zswap_pool *pool;

        assert_spin_locked(&zswap_pools_lock);

        list_for_each_entry_rcu(pool, &zswap_pools, list) {
                if (strcmp(pool->tfm_name, compressor))
                        continue;
                /* if we can't get it, it's about to be destroyed */
                if (!zswap_pool_tryget(pool))
                        continue;
                return pool;
        }

        return NULL;
}

static unsigned long zswap_max_pages(void)
{
        return totalram_pages() * zswap_max_pool_percent / 100;
}

static unsigned long zswap_accept_thr_pages(void)
{
        return zswap_max_pages() * zswap_accept_thr_percent / 100;
}

unsigned long zswap_total_pages(void)
{
        struct zswap_pool *pool;
        unsigned long total = 0;

        rcu_read_lock();
        list_for_each_entry_rcu(pool, &zswap_pools, list)
                total += zs_get_total_pages(pool->zs_pool);
        rcu_read_unlock();

        return total;
}

static bool zswap_check_limits(void)
{
        unsigned long cur_pages = zswap_total_pages();
        unsigned long max_pages = zswap_max_pages();

        if (cur_pages >= max_pages) {
                zswap_pool_limit_hit++;
                zswap_pool_reached_full = true;
        } else if (zswap_pool_reached_full &&
                   cur_pages <= zswap_accept_thr_pages()) {
                        zswap_pool_reached_full = false;
        }
        return zswap_pool_reached_full;
}

/*********************************
* param callbacks
**********************************/

static int zswap_compressor_param_set(const char *val, const struct kernel_param *kp)
{
        struct zswap_pool *pool, *put_pool = NULL;
        char *s = strstrip((char *)val);
        bool create_pool = false;
        int ret = 0;

        mutex_lock(&zswap_init_lock);
        switch (zswap_init_state) {
        case ZSWAP_UNINIT:
                /* Handled in zswap_setup() */
                ret = param_set_charp(s, kp);
                break;
        case ZSWAP_INIT_SUCCEED:
                if (!zswap_has_pool || strcmp(s, *(char **)kp->arg))
                        create_pool = true;
                break;
        case ZSWAP_INIT_FAILED:
                pr_err("can't set param, initialization failed\n");
                ret = -ENODEV;
        }
        mutex_unlock(&zswap_init_lock);

        if (!create_pool)
                return ret;

        if (!crypto_has_acomp(s, 0, 0)) {
                pr_err("compressor %s not available\n", s);
                return -ENOENT;
        }

        spin_lock_bh(&zswap_pools_lock);

        pool = zswap_pool_find_get(s);
        if (pool) {
                zswap_pool_debug("using existing", pool);
                WARN_ON(pool == zswap_pool_current());
                list_del_rcu(&pool->list);
        }

        spin_unlock_bh(&zswap_pools_lock);

        if (!pool)
                pool = zswap_pool_create(s);
        else {
                /*
                 * Restore the initial ref dropped by percpu_ref_kill()
                 * when the pool was decommissioned and switch it again
                 * to percpu mode.
                 */
                percpu_ref_resurrect(&pool->ref);

                /* Drop the ref from zswap_pool_find_get(). */
                zswap_pool_put(pool);
        }

        if (pool)
                ret = param_set_charp(s, kp);
        else
                ret = -EINVAL;

        spin_lock_bh(&zswap_pools_lock);

        if (!ret) {
                put_pool = zswap_pool_current();
                list_add_rcu(&pool->list, &zswap_pools);
                zswap_has_pool = true;
        } else if (pool) {
                /*
                 * Add the possibly pre-existing pool to the end of the pools
                 * list; if it's new (and empty) then it'll be removed and
                 * destroyed by the put after we drop the lock
                 */
                list_add_tail_rcu(&pool->list, &zswap_pools);
                put_pool = pool;
        }

        spin_unlock_bh(&zswap_pools_lock);

        /*
         * Drop the ref from either the old current pool,
         * or the new pool we failed to add
         */
        if (put_pool)
                percpu_ref_kill(&put_pool->ref);

        return ret;
}

static int zswap_enabled_param_set(const char *val,
                                   const struct kernel_param *kp)
{
        int ret = -ENODEV;

        /* if this is load-time (pre-init) param setting, only set param. */
        if (system_state != SYSTEM_RUNNING)
                return param_set_bool(val, kp);

        mutex_lock(&zswap_init_lock);
        switch (zswap_init_state) {
        case ZSWAP_UNINIT:
                if (zswap_setup())
                        break;
                fallthrough;
        case ZSWAP_INIT_SUCCEED:
                if (!zswap_has_pool)
                        pr_err("can't enable, no pool configured\n");
                else
                        ret = param_set_bool(val, kp);
                break;
        case ZSWAP_INIT_FAILED:
                pr_err("can't enable, initialization failed\n");
        }
        mutex_unlock(&zswap_init_lock);

        return ret;
}

/*********************************
* lru functions
**********************************/

/* should be called under RCU */
#ifdef CONFIG_MEMCG
static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
{
        return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL;
}
#else
static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
{
        return NULL;
}
#endif

static inline int entry_to_nid(struct zswap_entry *entry)
{
        return page_to_nid(virt_to_page(entry));
}

static void zswap_lru_add(struct zswap_entry *entry)
{
        int nid = entry_to_nid(entry);
        struct mem_cgroup *memcg;

        /*
         * Note that it is safe to use rcu_read_lock() here, even in the face of
         * concurrent memcg offlining:
         *
         * 1. list_lru_add() is called before list_lru_one is dead. The
         *    new entry will be reparented to memcg's parent's list_lru.
         * 2. list_lru_add() is called after list_lru_one is dead. The
         *    new entry will be added directly to memcg's parent's list_lru.
         *
         * Similar reasoning holds for list_lru_del().
         */
        rcu_read_lock();
        memcg = mem_cgroup_from_entry(entry);
        /* will always succeed */
        list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg);
        rcu_read_unlock();
}

static void zswap_lru_del(struct zswap_entry *entry)
{
        int nid = entry_to_nid(entry);
        struct mem_cgroup *memcg;

        rcu_read_lock();
        memcg = mem_cgroup_from_entry(entry);
        /* will always succeed */
        list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg);
        rcu_read_unlock();
}

void zswap_lruvec_state_init(struct lruvec *lruvec)
{
        atomic_long_set(&lruvec->zswap_lruvec_state.nr_disk_swapins, 0);
}

void zswap_folio_swapin(struct folio *folio)
{
        struct lruvec *lruvec;

        if (folio) {
                rcu_read_lock();
                lruvec = folio_lruvec(folio);
                atomic_long_inc(&lruvec->zswap_lruvec_state.nr_disk_swapins);
                rcu_read_unlock();
        }
}

/*
 * This function should be called when a memcg is being offlined.
 *
 * Since the global shrinker shrink_worker() may hold a reference
 * of the memcg, we must check and release the reference in
 * zswap_next_shrink.
 *
 * shrink_worker() must handle the case where this function releases
 * the reference of memcg being shrunk.
 */
void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg)
{
        /* lock out zswap shrinker walking memcg tree */
        spin_lock(&zswap_shrink_lock);
        if (zswap_next_shrink == memcg) {
                do {
                        zswap_next_shrink = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
                } while (zswap_next_shrink && !mem_cgroup_online(zswap_next_shrink));
        }
        spin_unlock(&zswap_shrink_lock);
}

/*********************************
* zswap entry functions
**********************************/
static struct kmem_cache *zswap_entry_cache;

static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp, int nid)
{
        struct zswap_entry *entry;
        entry = kmem_cache_alloc_node(zswap_entry_cache, gfp, nid);
        if (!entry)
                return NULL;
        return entry;
}

static void zswap_entry_cache_free(struct zswap_entry *entry)
{
        kmem_cache_free(zswap_entry_cache, entry);
}

/*
 * Carries out the common pattern of freeing an entry's zsmalloc allocation,
 * freeing the entry itself, and decrementing the number of stored pages.
 */
static void zswap_entry_free(struct zswap_entry *entry)
{
        zswap_lru_del(entry);
        zs_free(entry->pool->zs_pool, entry->handle);
        zswap_pool_put(entry->pool);
        if (entry->objcg) {
                obj_cgroup_uncharge_zswap(entry->objcg, entry->length);
                obj_cgroup_put(entry->objcg);
        }
        if (entry->length == PAGE_SIZE)
                atomic_long_dec(&zswap_stored_incompressible_pages);
        zswap_entry_cache_free(entry);
        atomic_long_dec(&zswap_stored_pages);
}

/*********************************
* compressed storage functions
**********************************/
static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
{
        struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
        struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
        int ret = -ENOMEM;

        /*
         * To handle cases where the CPU goes through online-offline-online
         * transitions, we return if the acomp_ctx has already been initialized.
         */
        if (acomp_ctx->acomp) {
                WARN_ON_ONCE(IS_ERR(acomp_ctx->acomp));
                return 0;
        }

        acomp_ctx->buffer = kmalloc_node(PAGE_SIZE, GFP_KERNEL, cpu_to_node(cpu));
        if (!acomp_ctx->buffer)
                return ret;

        /*
         * In case of an error, crypto_alloc_acomp_node() returns an
         * error pointer, never NULL.
         */
        acomp_ctx->acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));
        if (IS_ERR(acomp_ctx->acomp)) {
                pr_err("could not alloc crypto acomp %s : %pe\n",
                                pool->tfm_name, acomp_ctx->acomp);
                ret = PTR_ERR(acomp_ctx->acomp);
                goto fail;
        }

        /* acomp_request_alloc() returns NULL in case of an error. */
        acomp_ctx->req = acomp_request_alloc(acomp_ctx->acomp);
        if (!acomp_ctx->req) {
                pr_err("could not alloc crypto acomp_request %s\n",
                       pool->tfm_name);
                goto fail;
        }

        crypto_init_wait(&acomp_ctx->wait);

        /*
         * if the backend of acomp is async zip, crypto_req_done() will wakeup
         * crypto_wait_req(); if the backend of acomp is scomp, the callback
         * won't be called, crypto_wait_req() will return without blocking.
         */
        acomp_request_set_callback(acomp_ctx->req, CRYPTO_TFM_REQ_MAY_BACKLOG,
                                   crypto_req_done, &acomp_ctx->wait);

        mutex_init(&acomp_ctx->mutex);
        return 0;

fail:
        acomp_ctx_free(acomp_ctx);
        return ret;
}

static bool zswap_compress(struct page *page, struct zswap_entry *entry,
                           struct zswap_pool *pool)
{
        struct crypto_acomp_ctx *acomp_ctx;
        struct scatterlist input, output;
        int comp_ret = 0, alloc_ret = 0;
        unsigned int dlen = PAGE_SIZE;
        unsigned long handle;
        gfp_t gfp;
        u8 *dst;
        bool mapped = false;

        acomp_ctx = raw_cpu_ptr(pool->acomp_ctx);
        mutex_lock(&acomp_ctx->mutex);

        dst = acomp_ctx->buffer;
        sg_init_table(&input, 1);
        sg_set_page(&input, page, PAGE_SIZE, 0);

        sg_init_one(&output, dst, PAGE_SIZE);
        acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);

        /*
         * it maybe looks a little bit silly that we send an asynchronous request,
         * then wait for its completion synchronously. This makes the process look
         * synchronous in fact.
         * Theoretically, acomp supports users send multiple acomp requests in one
         * acomp instance, then get those requests done simultaneously. but in this
         * case, zswap actually does store and load page by page, there is no
         * existing method to send the second page before the first page is done
         * in one thread doing zswap.
         * but in different threads running on different cpu, we have different
         * acomp instance, so multiple threads can do (de)compression in parallel.
         */
        comp_ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);
        dlen = acomp_ctx->req->dlen;

        /*
         * If a page cannot be compressed into a size smaller than PAGE_SIZE,
         * save the content as is without a compression, to keep the LRU order
         * of writebacks.  If writeback is disabled, reject the page since it
         * only adds metadata overhead.  swap_writeout() will put the page back
         * to the active LRU list in the case.
         */
        if (comp_ret || !dlen || dlen >= PAGE_SIZE) {
                rcu_read_lock();
                if (!mem_cgroup_zswap_writeback_enabled(
                                        folio_memcg(page_folio(page)))) {
                        rcu_read_unlock();
                        comp_ret = comp_ret ? comp_ret : -EINVAL;
                        goto unlock;
                }
                rcu_read_unlock();
                comp_ret = 0;
                dlen = PAGE_SIZE;
                dst = kmap_local_page(page);
                mapped = true;
        }

        gfp = GFP_NOWAIT | __GFP_NORETRY | __GFP_HIGHMEM | __GFP_MOVABLE;
        handle = zs_malloc(pool->zs_pool, dlen, gfp, page_to_nid(page));
        if (IS_ERR_VALUE(handle)) {
                alloc_ret = PTR_ERR((void *)handle);
                goto unlock;
        }

        zs_obj_write(pool->zs_pool, handle, dst, dlen);
        entry->handle = handle;
        entry->length = dlen;

unlock:
        if (mapped)
                kunmap_local(dst);
        if (comp_ret == -ENOSPC || alloc_ret == -ENOSPC)
                zswap_reject_compress_poor++;
        else if (comp_ret)
                zswap_reject_compress_fail++;
        else if (alloc_ret)
                zswap_reject_alloc_fail++;

        mutex_unlock(&acomp_ctx->mutex);
        return comp_ret == 0 && alloc_ret == 0;
}

static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio)
{
        struct zswap_pool *pool = entry->pool;
        struct scatterlist input[2]; /* zsmalloc returns an SG list 1-2 entries */
        struct scatterlist output;
        struct crypto_acomp_ctx *acomp_ctx;
        int ret = 0, dlen;

        acomp_ctx = raw_cpu_ptr(pool->acomp_ctx);
        mutex_lock(&acomp_ctx->mutex);
        zs_obj_read_sg_begin(pool->zs_pool, entry->handle, input, entry->length);

        /* zswap entries of length PAGE_SIZE are not compressed. */
        if (entry->length == PAGE_SIZE) {
                void *dst;

                WARN_ON_ONCE(input->length != PAGE_SIZE);

                dst = kmap_local_folio(folio, 0);
                memcpy_from_sglist(dst, input, 0, PAGE_SIZE);
                dlen = PAGE_SIZE;
                kunmap_local(dst);
                flush_dcache_folio(folio);
        } else {
                sg_init_table(&output, 1);
                sg_set_folio(&output, folio, PAGE_SIZE, 0);
                acomp_request_set_params(acomp_ctx->req, input, &output,
                                         entry->length, PAGE_SIZE);
                ret = crypto_acomp_decompress(acomp_ctx->req);
                ret = crypto_wait_req(ret, &acomp_ctx->wait);
                dlen = acomp_ctx->req->dlen;
        }

        zs_obj_read_sg_end(pool->zs_pool, entry->handle);
        mutex_unlock(&acomp_ctx->mutex);

        if (!ret && dlen == PAGE_SIZE)
                return true;

        zswap_decompress_fail++;
        pr_alert_ratelimited("Decompression error from zswap (%d:%lu %s %u->%d)\n",
                                                swp_type(entry->swpentry),
                                                swp_offset(entry->swpentry),
                                                entry->pool->tfm_name,
                                                entry->length, dlen);
        return false;
}

/*********************************
* writeback code
**********************************/
/*
 * Attempts to free an entry by adding a folio to the swap cache,
 * decompressing the entry data into the folio, and issuing a
 * bio write to write the folio back to the swap device.
 *
 * This can be thought of as a "resumed writeback" of the folio
 * to the swap device.  We are basically resuming the same swap
 * writeback path that was intercepted with the zswap_store()
 * in the first place.  After the folio has been decompressed into
 * the swap cache, the compressed version stored by zswap can be
 * freed.
 */
static int zswap_writeback_entry(struct zswap_entry *entry,
                                 swp_entry_t swpentry)
{
        struct xarray *tree;
        pgoff_t offset = swp_offset(swpentry);
        struct folio *folio;
        struct mempolicy *mpol;
        struct swap_info_struct *si;
        struct swap_io_ctx ctx = {};
        int ret = 0;

        /* try to allocate swap cache folio */
        si = get_swap_device(swpentry);
        if (!si)
                return -EEXIST;

        mpol = get_task_policy(current);
        folio = swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol,
                                       NO_INTERLEAVE_INDEX);
        put_swap_device(si);

        /*
         * Swap cache allocation might fail due to OOM, or the entry
         * may already be cached due to concurrent swapin or have been
         * freed. If already cached, a concurrent swapin made the folio
         * hot, so skip it. For the unlikely concurrent shrinker case,
         * it will be unlinked and freed when invalidated anyway.
         */
        if (IS_ERR(folio))
                return PTR_ERR(folio);

        /*
         * folio is locked, and the swapcache is now secured against
         * concurrent swapping to and from the slot, and concurrent
         * swapoff so we can safely dereference the zswap tree here.
         * Verify that the swap entry hasn't been invalidated and recycled
         * behind our backs, to avoid overwriting a new swap folio with
         * old compressed data. Only when this is successful can the entry
         * be dereferenced.
         */
        tree = swap_zswap_tree(swpentry);
        if (entry != xa_load(tree, offset)) {
                ret = -ENOMEM;
                goto out;
        }

        if (!zswap_decompress(entry, folio)) {
                ret = -EIO;
                goto out;
        }

        xa_erase(tree, offset);

        count_vm_event(ZSWPWB);
        if (entry->objcg)
                count_objcg_events(entry->objcg, ZSWPWB, 1);

        zswap_entry_free(entry);

        /* folio is up to date */
        folio_mark_uptodate(folio);

        /* move it to the tail of the inactive list after end_writeback */
        folio_set_reclaim(folio);

        /* start writeback */
        __swap_writepage(&ctx, folio);
        swap_write_submit(&ctx);

out:
        if (ret) {
                swap_cache_del_folio(folio);
                folio_unlock(folio);
        }
        folio_put(folio);
        return ret;
}

/*********************************
* shrinker functions
**********************************/
/*
 * The dynamic shrinker is modulated by the following factors:
 *
 * 1. Each zswap entry has a referenced bit, which the shrinker unsets (giving
 *    the entry a second chance) before rotating it in the LRU list. If the
 *    entry is considered again by the shrinker, with its referenced bit unset,
 *    it is written back. The writeback rate as a result is dynamically
 *    adjusted by the pool activities - if the pool is dominated by new entries
 *    (i.e lots of recent zswapouts), these entries will be protected and
 *    the writeback rate will slow down. On the other hand, if the pool has a
 *    lot of stagnant entries, these entries will be reclaimed immediately,
 *    effectively increasing the writeback rate.
 *
 * 2. Swapins counter: If we observe swapins, it is a sign that we are
 *    overshrinking and should slow down. We maintain a swapins counter, which
 *    is consumed and subtract from the number of eligible objects on the LRU
 *    in zswap_shrinker_count().
 *
 * 3. Compression ratio. The better the workload compresses, the less gains we
 *    can expect from writeback. We scale down the number of objects available
 *    for reclaim by this ratio.
 */
static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_one *l,
                                       void *arg)
{
        struct zswap_entry *entry = container_of(item, struct zswap_entry, lru);
        bool *encountered_page_in_swapcache = (bool *)arg;
        swp_entry_t swpentry;
        enum lru_status ret = LRU_REMOVED_RETRY;
        int writeback_result;

        /*
         * Second chance algorithm: if the entry has its referenced bit set, give it
         * a second chance. Only clear the referenced bit and rotate it in the
         * zswap's LRU list.
         */
        if (entry->referenced) {
                entry->referenced = false;
                return LRU_ROTATE;
        }

        /*
         * As soon as we drop the LRU lock, the entry can be freed by
         * a concurrent invalidation. This means the following:
         *
         * 1. We extract the swp_entry_t to the stack, allowing
         *    zswap_writeback_entry() to pin the swap entry and
         *    then validate the zswap entry against that swap entry's
         *    tree using pointer value comparison. Only when that
         *    is successful can the entry be dereferenced.
         *
         * 2. Usually, objects are taken off the LRU for reclaim. In
         *    this case this isn't possible, because if reclaim fails
         *    for whatever reason, we have no means of knowing if the
         *    entry is alive to put it back on the LRU.
         *
         *    So rotate it before dropping the lock. If the entry is
         *    written back or invalidated, the free path will unlink
         *    it. For failures, rotation is the right thing as well.
         *
         *    Temporary failures, where the same entry should be tried
         *    again immediately, almost never happen for this shrinker.
         *    We don't do any trylocking; -ENOMEM comes closest,
         *    but that's extremely rare and doesn't happen spuriously
         *    either. Don't bother distinguishing this case.
         */
        list_move_tail(item, &l->list);

        /*
         * Once the lru lock is dropped, the entry might get freed. The
         * swpentry is copied to the stack, and entry isn't deref'd again
         * until the entry is verified to still be alive in the tree.
         */
        swpentry = entry->swpentry;

        /*
         * It's safe to drop the lock here because we return either
         * LRU_REMOVED_RETRY, LRU_RETRY or LRU_STOP.
         */
        spin_unlock(&l->lock);

        writeback_result = zswap_writeback_entry(entry, swpentry);

        if (writeback_result) {
                zswap_reject_reclaim_fail++;
                ret = LRU_RETRY;

                /*
                 * Encountering a page already in swap cache is a sign that we are shrinking
                 * into the warmer region. We should terminate shrinking (if we're in the dynamic
                 * shrinker context).
                 */
                if (writeback_result == -EEXIST && encountered_page_in_swapcache) {
                        ret = LRU_STOP;
                        *encountered_page_in_swapcache = true;
                }
        } else {
                zswap_written_back_pages++;
        }

        return ret;
}

static unsigned long zswap_shrinker_scan(struct shrinker *shrinker,
                struct shrink_control *sc)
{
        unsigned long shrink_ret;
        bool encountered_page_in_swapcache = false;

        if (!zswap_shrinker_enabled ||
                        !mem_cgroup_zswap_writeback_enabled(sc->memcg)) {
                sc->nr_scanned = 0;
                return SHRINK_STOP;
        }

        shrink_ret = list_lru_shrink_walk(&zswap_list_lru, sc, &shrink_memcg_cb,
                &encountered_page_in_swapcache);

        if (encountered_page_in_swapcache)
                return SHRINK_STOP;

        return shrink_ret ? shrink_ret : SHRINK_STOP;
}

static unsigned long zswap_shrinker_count(struct shrinker *shrinker,
                struct shrink_control *sc)
{
        struct mem_cgroup *memcg = sc->memcg;
        struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid));
        atomic_long_t *nr_disk_swapins =
                &lruvec->zswap_lruvec_state.nr_disk_swapins;
        unsigned long nr_backing, nr_stored, nr_freeable, nr_disk_swapins_cur,
                nr_remain;

        if (!zswap_shrinker_enabled || !mem_cgroup_zswap_writeback_enabled(memcg))
                return 0;

        /*
         * The shrinker resumes swap writeback, which will enter block
         * and may enter fs. XXX: Harmonize with vmscan.c __GFP_FS
         * rules (may_enter_fs()), which apply on a per-folio basis.
         */
        if (!gfp_has_io_fs(sc->gfp_mask))
                return 0;

        /*
         * For memcg, use the cgroup-wide ZSWAP stats since we don't
         * have them per-node and thus per-lruvec. Careful if memcg is
         * runtime-disabled: we can get sc->memcg == NULL, which is ok
         * for the lruvec, but not for memcg_page_state().
         *
         * Without memcg, use the zswap pool-wide metrics.
         */
        if (!mem_cgroup_disabled()) {
                mem_cgroup_flush_stats_ratelimited(memcg);
                nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT;
                nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED);
        } else {
                nr_backing = zswap_total_pages();
                nr_stored = atomic_long_read(&zswap_stored_pages);
        }

        if (!nr_stored)
                return 0;

        nr_freeable = list_lru_shrink_count(&zswap_list_lru, sc);
        if (!nr_freeable)
                return 0;

        /*
         * Subtract from the lru size the number of pages that are recently swapped
         * in from disk. The idea is that had we protect the zswap's LRU by this
         * amount of pages, these disk swapins would not have happened.
         */
        nr_disk_swapins_cur = atomic_long_read(nr_disk_swapins);
        do {
                if (nr_freeable >= nr_disk_swapins_cur)
                        nr_remain = 0;
                else
                        nr_remain = nr_disk_swapins_cur - nr_freeable;
        } while (!atomic_long_try_cmpxchg(
                nr_disk_swapins, &nr_disk_swapins_cur, nr_remain));

        nr_freeable -= nr_disk_swapins_cur - nr_remain;
        if (!nr_freeable)
                return 0;

        /*
         * Scale the number of freeable pages by the memory saving factor.
         * This ensures that the better zswap compresses memory, the fewer
         * pages we will evict to swap (as it will otherwise incur IO for
         * relatively small memory saving).
         */
        return mult_frac(nr_freeable, nr_backing, nr_stored);
}

static struct shrinker *zswap_alloc_shrinker(void)
{
        struct shrinker *shrinker;

        shrinker =
                shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE, "mm-zswap");
        if (!shrinker)
                return NULL;

        shrinker->scan_objects = zswap_shrinker_scan;
        shrinker->count_objects = zswap_shrinker_count;
        shrinker->batch = 0;
        shrinker->seeks = DEFAULT_SEEKS;
        return shrinker;
}

/*
 * Scan up to SWAP_CLUSTER_MAX pages on each per-node zswap LRU of @memcg
 * and write back the reclaimable ones.
 *
 * Return: 0 if at least one entry was written back, -EAGAIN if entries
 * were scanned but none could be written back, or -ENOENT if @memcg has
 * writeback disabled, is a zombie cgroup, or has empty zswap LRUs.
 */
static int shrink_memcg(struct mem_cgroup *memcg)
{
        int nid, shrunk = 0, scanned = 0;

        if (!mem_cgroup_zswap_writeback_enabled(memcg))
                return -ENOENT;

        /*
         * Skip zombies because their LRUs are reparented and we would be
         * reclaiming from the parent instead of the dead memcg.
         */
        if (memcg && !mem_cgroup_online(memcg))
                return -ENOENT;

        for_each_node_state(nid, N_NORMAL_MEMORY) {
                unsigned long nr_to_walk = SWAP_CLUSTER_MAX;

                shrunk += list_lru_walk_one(&zswap_list_lru, nid, memcg,
                                            &shrink_memcg_cb, NULL, &nr_to_walk);
                scanned += SWAP_CLUSTER_MAX - nr_to_walk;
        }

        /* Nothing was scanned: every LRU under @memcg was empty. */
        if (!scanned)
                return -ENOENT;

        return shrunk ? 0 : -EAGAIN;
}

static void shrink_worker(struct work_struct *w)
{
        struct mem_cgroup *memcg;
        int ret, failures = 0, attempts = 0;
        unsigned long thr;

        /* Reclaim down to the accept threshold */
        thr = zswap_accept_thr_pages();

        /*
         * Global reclaim will select cgroup in a round-robin fashion from all
         * online memcgs, but memcgs that have no pages in zswap and
         * writeback-disabled memcgs (memory.zswap.writeback=0) are not
         * candidates for shrinking.
         *
         * Shrinking will be aborted if we encounter the following
         * MAX_RECLAIM_RETRIES times:
         * - No writeback-candidate memcgs found in a memcg tree walk.
         * - Shrinking a writeback-candidate memcg failed.
         *
         * We save iteration cursor memcg into zswap_next_shrink,
         * which can be modified by the offline memcg cleaner
         * zswap_memcg_offline_cleanup().
         *
         * Since the offline cleaner is called only once, we cannot leave an
         * offline memcg reference in zswap_next_shrink.
         * We can rely on the cleaner only if we get online memcg under lock.
         *
         * If we get an offline memcg, we cannot determine if the cleaner has
         * already been called or will be called later. We must put back the
         * reference before returning from this function. Otherwise, the
         * offline memcg left in zswap_next_shrink will hold the reference
         * until the next run of shrink_worker().
         */
        do {
                /*
                 * Start shrinking from the next memcg after zswap_next_shrink.
                 * When the offline cleaner has already advanced the cursor,
                 * advancing the cursor here overlooks one memcg, but this
                 * should be negligibly rare.
                 *
                 * If we get an online memcg, keep the extra reference in case
                 * the original one obtained by mem_cgroup_iter() is dropped by
                 * zswap_memcg_offline_cleanup() while we are shrinking the
                 * memcg.
                 */
                spin_lock(&zswap_shrink_lock);
                do {
                        memcg = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
                        zswap_next_shrink = memcg;
                } while (memcg && !mem_cgroup_tryget_online(memcg));
                spin_unlock(&zswap_shrink_lock);

                /*
                 * A NULL memcg ends a full hierarchy pass (except when memcg is
                 * disabled, where it is always NULL: fall through to the root LRU).
                 * Count a failure only if the last pass found no candidates.
                 */
                if (!memcg && !mem_cgroup_disabled()) {
                        if (!attempts && ++failures == MAX_RECLAIM_RETRIES)
                                break;

                        attempts = 0;
                        goto resched;
                }

                ret = shrink_memcg(memcg);
                /* drop the extra reference */
                mem_cgroup_put(memcg);

                /*
                 * There are no writeback-candidate pages in the memcg.
                 * This is not an issue as long as we can find another memcg
                 * with pages in zswap. Skip this without incrementing attempts
                 * and failures.
                 */
                if (ret == -ENOENT)
                        goto resched;
                ++attempts;

                if (ret && ++failures == MAX_RECLAIM_RETRIES)
                        break;
resched:
                cond_resched();
        } while (zswap_total_pages() > thr);
}

/*********************************
* main API
**********************************/

static bool zswap_store_page(struct page *page,
                             struct obj_cgroup *objcg,
                             struct zswap_pool *pool)
{
        swp_entry_t page_swpentry = page_swap_entry(page);
        struct zswap_entry *entry, *old;

        /* allocate entry */
        entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page));
        if (!entry) {
                zswap_reject_kmemcache_fail++;
                return false;
        }

        if (!zswap_compress(page, entry, pool))
                goto compress_failed;

        old = xa_store(swap_zswap_tree(page_swpentry),
                       swp_offset(page_swpentry),
                       entry, GFP_KERNEL);
        if (xa_is_err(old)) {
                int err = xa_err(old);

                WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err);
                zswap_reject_alloc_fail++;
                goto store_failed;
        }

        /*
         * We may have had an existing entry that became stale when
         * the folio was redirtied and now the new version is being
         * swapped out. Get rid of the old.
         */
        if (old)
                zswap_entry_free(old);

        /*
         * The entry is successfully compressed and stored in the tree, there is
         * no further possibility of failure. Grab refs to the pool and objcg,
         * charge zswap memory, and increment zswap_stored_pages.
         * The opposite actions will be performed by zswap_entry_free()
         * when the entry is removed from the tree.
         */
        zswap_pool_get(pool);
        if (objcg) {
                obj_cgroup_get(objcg);
                obj_cgroup_charge_zswap(objcg, entry->length);
        }
        atomic_long_inc(&zswap_stored_pages);
        if (entry->length == PAGE_SIZE)
                atomic_long_inc(&zswap_stored_incompressible_pages);

        /*
         * We finish initializing the entry while it's already in xarray.
         * This is safe because:
         *
         * 1. Concurrent stores and invalidations are excluded by folio lock.
         *
         * 2. Writeback is excluded by the entry not being on the LRU yet.
         *    The publishing order matters to prevent writeback from seeing
         *    an incoherent entry.
         */
        entry->pool = pool;
        entry->swpentry = page_swpentry;
        entry->objcg = objcg;
        entry->referenced = true;
        if (entry->length) {
                INIT_LIST_HEAD(&entry->lru);
                zswap_lru_add(entry);
        }

        return true;

store_failed:
        zs_free(pool->zs_pool, entry->handle);
compress_failed:
        zswap_entry_cache_free(entry);
        return false;
}

bool zswap_store(struct folio *folio)
{
        long nr_pages = folio_nr_pages(folio);
        swp_entry_t swp = folio->swap;
        struct obj_cgroup *objcg = NULL;
        struct mem_cgroup *memcg = NULL;
        struct zswap_pool *pool;
        bool ret = false;
        long index;

        VM_WARN_ON_ONCE(!folio_test_locked(folio));
        VM_WARN_ON_ONCE(!folio_test_swapcache(folio));

        if (!zswap_enabled)
                goto check_old;

        objcg = get_obj_cgroup_from_folio(folio);
        if (objcg && !obj_cgroup_may_zswap(objcg)) {
                memcg = get_mem_cgroup_from_objcg(objcg);
                if (shrink_memcg(memcg)) {
                        mem_cgroup_put(memcg);
                        goto put_objcg;
                }
                mem_cgroup_put(memcg);
        }

        if (zswap_check_limits())
                goto put_objcg;

        pool = zswap_pool_current_get();
        if (!pool)
                goto put_objcg;

        if (objcg) {
                memcg = get_mem_cgroup_from_objcg(objcg);
                if (memcg_list_lru_alloc(memcg, &zswap_list_lru, GFP_KERNEL)) {
                        mem_cgroup_put(memcg);
                        goto put_pool;
                }
                mem_cgroup_put(memcg);
        }

        for (index = 0; index < nr_pages; ++index) {
                struct page *page = folio_page(folio, index);

                if (!zswap_store_page(page, objcg, pool))
                        goto put_pool;
        }

        if (objcg)
                count_objcg_events(objcg, ZSWPOUT, nr_pages);

        count_vm_events(ZSWPOUT, nr_pages);

        ret = true;

put_pool:
        zswap_pool_put(pool);
put_objcg:
        obj_cgroup_put(objcg);
        if (!ret && zswap_pool_reached_full)
                queue_work(shrink_wq, &zswap_shrink_work);
check_old:
        /*
         * If the zswap store fails or zswap is disabled, we must invalidate
         * the possibly stale entries which were previously stored at the
         * offsets corresponding to each page of the folio. Otherwise,
         * writeback could overwrite the new data in the swapfile.
         */
        if (!ret) {
                unsigned type = swp_type(swp);
                pgoff_t offset = swp_offset(swp);
                struct zswap_entry *entry;
                struct xarray *tree;

                for (index = 0; index < nr_pages; ++index) {
                        tree = swap_zswap_tree(swp_entry(type, offset + index));
                        entry = xa_erase(tree, offset + index);
                        if (entry)
                                zswap_entry_free(entry);
                }
        }

        return ret;
}

/**
 * zswap_load() - load a folio from zswap
 * @folio: folio to load
 *
 * Return: 0 on success, with the folio unlocked and marked up-to-date, or one
 * of the following error codes:
 *
 *  -EIO: if the swapped out content was in zswap, but could not be loaded
 *  into the page due to a decompression failure. The folio is unlocked, but
 *  NOT marked up-to-date, so that an IO error is emitted (e.g. do_swap_page()
 *  will SIGBUS).
 *
 *  -EINVAL: if the swapped out content was in zswap, but the page belongs
 *  to a large folio, which is not supported by zswap. The folio is unlocked,
 *  but NOT marked up-to-date, so that an IO error is emitted (e.g.
 *  do_swap_page() will SIGBUS).
 *
 *  -ENOENT: if the swapped out content was not in zswap. The folio remains
 *  locked on return.
 */
int zswap_load(struct folio *folio)
{
        swp_entry_t swp = folio->swap;
        pgoff_t offset = swp_offset(swp);
        struct xarray *tree = swap_zswap_tree(swp);
        struct zswap_entry *entry;

        VM_WARN_ON_ONCE(!folio_test_locked(folio));
        VM_WARN_ON_ONCE(!folio_test_swapcache(folio));

        if (zswap_never_enabled())
                return -ENOENT;

        /*
         * Large folios should not be swapped in while zswap is being used, as
         * they are not properly handled. Zswap does not properly load large
         * folios, and a large folio may only be partially in zswap.
         */
        if (WARN_ON_ONCE(folio_test_large(folio))) {
                folio_unlock(folio);
                return -EINVAL;
        }

        entry = xa_load(tree, offset);
        if (!entry)
                return -ENOENT;

        if (!zswap_decompress(entry, folio)) {
                folio_unlock(folio);
                return -EIO;
        }

        folio_mark_uptodate(folio);

        count_vm_event(ZSWPIN);
        if (entry->objcg)
                count_objcg_events(entry->objcg, ZSWPIN, 1);

        /*
         * We are reading into the swapcache, invalidate zswap entry.
         * The swapcache is the authoritative owner of the page and
         * its mappings, and the pressure that results from having two
         * in-memory copies outweighs any benefits of caching the
         * compression work.
         */
        folio_mark_dirty(folio);
        xa_erase(tree, offset);
        zswap_entry_free(entry);

        folio_unlock(folio);
        return 0;
}

void zswap_invalidate(swp_entry_t swp)
{
        pgoff_t offset = swp_offset(swp);
        struct xarray *tree = swap_zswap_tree(swp);
        struct zswap_entry *entry;

        if (xa_empty(tree))
                return;

        entry = xa_erase(tree, offset);
        if (entry)
                zswap_entry_free(entry);
}

int zswap_swapon(int type, unsigned long nr_pages)
{
        struct xarray *trees, *tree;
        unsigned int nr, i;

        nr = DIV_ROUND_UP(nr_pages, ZSWAP_ADDRESS_SPACE_PAGES);
        trees = kvzalloc_objs(*tree, nr);
        if (!trees) {
                pr_err("alloc failed, zswap disabled for swap type %d\n", type);
                return -ENOMEM;
        }

        for (i = 0; i < nr; i++)
                xa_init(trees + i);

        nr_zswap_trees[type] = nr;
        zswap_trees[type] = trees;
        return 0;
}

void zswap_swapoff(int type)
{
        struct xarray *trees = zswap_trees[type];
        unsigned int i;

        if (!trees)
                return;

        /* try_to_unuse() invalidated all the entries already */
        for (i = 0; i < nr_zswap_trees[type]; i++)
                WARN_ON_ONCE(!xa_empty(trees + i));

        kvfree(trees);
        nr_zswap_trees[type] = 0;
        zswap_trees[type] = NULL;
}

/*********************************
* debugfs functions
**********************************/
#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>

static struct dentry *zswap_debugfs_root;

static int debugfs_get_total_size(void *data, u64 *val)
{
        *val = zswap_total_pages() * PAGE_SIZE;
        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(total_size_fops, debugfs_get_total_size, NULL, "%llu\n");

static int debugfs_get_stored_pages(void *data, u64 *val)
{
        *val = atomic_long_read(&zswap_stored_pages);
        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(stored_pages_fops, debugfs_get_stored_pages, NULL, "%llu\n");

static int debugfs_get_stored_incompressible_pages(void *data, u64 *val)
{
        *val = atomic_long_read(&zswap_stored_incompressible_pages);
        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(stored_incompressible_pages_fops,
                debugfs_get_stored_incompressible_pages, NULL, "%llu\n");

static int zswap_debugfs_init(void)
{
        if (!debugfs_initialized())
                return -ENODEV;

        zswap_debugfs_root = debugfs_create_dir("zswap", NULL);

        debugfs_create_u64("pool_limit_hit", 0444,
                           zswap_debugfs_root, &zswap_pool_limit_hit);
        debugfs_create_u64("reject_reclaim_fail", 0444,
                           zswap_debugfs_root, &zswap_reject_reclaim_fail);
        debugfs_create_u64("reject_alloc_fail", 0444,
                           zswap_debugfs_root, &zswap_reject_alloc_fail);
        debugfs_create_u64("reject_kmemcache_fail", 0444,
                           zswap_debugfs_root, &zswap_reject_kmemcache_fail);
        debugfs_create_u64("reject_compress_fail", 0444,
                           zswap_debugfs_root, &zswap_reject_compress_fail);
        debugfs_create_u64("reject_compress_poor", 0444,
                           zswap_debugfs_root, &zswap_reject_compress_poor);
        debugfs_create_u64("decompress_fail", 0444,
                           zswap_debugfs_root, &zswap_decompress_fail);
        debugfs_create_u64("written_back_pages", 0444,
                           zswap_debugfs_root, &zswap_written_back_pages);
        debugfs_create_file("pool_total_size", 0444,
                            zswap_debugfs_root, NULL, &total_size_fops);
        debugfs_create_file("stored_pages", 0444,
                            zswap_debugfs_root, NULL, &stored_pages_fops);
        debugfs_create_file("stored_incompressible_pages", 0444,
                            zswap_debugfs_root, NULL,
                            &stored_incompressible_pages_fops);

        return 0;
}
#else
static int zswap_debugfs_init(void)
{
        return 0;
}
#endif

/*********************************
* module init and exit
**********************************/
static int zswap_setup(void)
{
        struct zswap_pool *pool;
        int ret;

        zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
        if (!zswap_entry_cache) {
                pr_err("entry cache creation failed\n");
                goto cache_fail;
        }

        ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
                                      "mm/zswap_pool:prepare",
                                      zswap_cpu_comp_prepare,
                                      NULL);
        if (ret)
                goto hp_fail;

        shrink_wq = alloc_workqueue("zswap-shrink",
                        WQ_UNBOUND|WQ_MEM_RECLAIM, 1);
        if (!shrink_wq)
                goto shrink_wq_fail;

        zswap_shrinker = zswap_alloc_shrinker();
        if (!zswap_shrinker)
                goto shrinker_fail;
        if (list_lru_init_memcg(&zswap_list_lru, zswap_shrinker))
                goto lru_fail;
        shrinker_register(zswap_shrinker);

        INIT_WORK(&zswap_shrink_work, shrink_worker);

        pool = __zswap_pool_create_fallback();
        if (pool) {
                pr_info("loaded using pool %s\n", pool->tfm_name);
                list_add(&pool->list, &zswap_pools);
                zswap_has_pool = true;
                static_branch_enable(&zswap_ever_enabled);
        } else {
                pr_err("pool creation failed\n");
                zswap_enabled = false;
        }

        if (zswap_debugfs_init())
                pr_warn("debugfs initialization failed\n");
        zswap_init_state = ZSWAP_INIT_SUCCEED;
        return 0;

lru_fail:
        shrinker_free(zswap_shrinker);
shrinker_fail:
        destroy_workqueue(shrink_wq);
shrink_wq_fail:
        cpuhp_remove_multi_state(CPUHP_MM_ZSWP_POOL_PREPARE);
hp_fail:
        kmem_cache_destroy(zswap_entry_cache);
cache_fail:
        /* if built-in, we aren't unloaded on failure; don't allow use */
        zswap_init_state = ZSWAP_INIT_FAILED;
        zswap_enabled = false;
        return -ENOMEM;
}

static int __init zswap_init(void)
{
        if (!zswap_enabled)
                return 0;
        return zswap_setup();
}
/* must be late so crypto has time to come up */
late_initcall(zswap_init);

MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
MODULE_DESCRIPTION("Compressed cache for swap pages");
















































































































































































































































































































   27 



   17 



    1 














   37 































   17 
   20 









   37 


























   10 

    5 


    5 































   50 








   50 


   50 






   50 

   50 











































































































































































































































   11 


   11 







    4 


    2 

    3 
    2 










    5 
    5 




    5 

























    1 


    5 










    5 


    5 

    5 

































































    6 






    1 










    1 







    1 
















    3 



















    3 






























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
// SPDX-License-Identifier: GPL-2.0-only
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Implementation of the Transmission Control Protocol(TCP).
 *
 * Authors:        Ross Biro
 *                Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *                Mark Evans, <evansmp@uhura.aston.ac.uk>
 *                Corey Minyard <wf-rch!minyard@relay.EU.net>
 *                Florian La Roche, <flla@stud.uni-sb.de>
 *                Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
 *                Linus Torvalds, <torvalds@cs.helsinki.fi>
 *                Alan Cox, <gw4pts@gw4pts.ampr.org>
 *                Matthew Dillon, <dillon@apollo.west.oic.com>
 *                Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 *                Jorge Cwik, <jorge@laser.satlink.net>
 */

#include <net/tcp.h>
#include <net/tcp_ecn.h>
#include <net/xfrm.h>
#include <net/busy_poll.h>
#include <net/rstreason.h>
#include <net/psp.h>

static bool tcp_in_window(u32 seq, u32 end_seq, u32 s_win, u32 e_win)
{
        if (seq == s_win)
                return true;
        if (after(end_seq, s_win) && before(seq, e_win))
                return true;
        return seq == e_win && seq == end_seq;
}

static enum tcp_tw_status
tcp_timewait_check_oow_rate_limit(struct inet_timewait_sock *tw,
                                  const struct sk_buff *skb, int mib_idx)
{
        struct tcp_timewait_sock *tcptw = tcp_twsk((struct sock *)tw);

        if (!tcp_oow_rate_limited(twsk_net(tw), skb, mib_idx,
                                  &tcptw->tw_last_oow_ack_time)) {
                /* Send ACK. Note, we do not put the bucket,
                 * it will be released by caller.
                 */
                return TCP_TW_ACK_OOW;
        }

        /* We are rate-limiting, so just release the tw sock and drop skb. */
        inet_twsk_put(tw);
        return TCP_TW_SUCCESS;
}

static void twsk_rcv_nxt_update(struct tcp_timewait_sock *tcptw, u32 seq,
                                u32 rcv_nxt)
{
#ifdef CONFIG_TCP_AO
        struct tcp_ao_info *ao;

        ao = rcu_dereference(tcptw->ao_info);
        if (unlikely(ao && seq < rcv_nxt))
                WRITE_ONCE(ao->rcv_sne, ao->rcv_sne + 1);
#endif
        WRITE_ONCE(tcptw->tw_rcv_nxt, seq);
}

/*
 * * Main purpose of TIME-WAIT state is to close connection gracefully,
 *   when one of ends sits in LAST-ACK or CLOSING retransmitting FIN
 *   (and, probably, tail of data) and one or more our ACKs are lost.
 * * What is TIME-WAIT timeout? It is associated with maximal packet
 *   lifetime in the internet, which results in wrong conclusion, that
 *   it is set to catch "old duplicate segments" wandering out of their path.
 *   It is not quite correct. This timeout is calculated so that it exceeds
 *   maximal retransmission timeout enough to allow to lose one (or more)
 *   segments sent by peer and our ACKs. This time may be calculated from RTO.
 * * When TIME-WAIT socket receives RST, it means that another end
 *   finally closed and we are allowed to kill TIME-WAIT too.
 * * Second purpose of TIME-WAIT is catching old duplicate segments.
 *   Well, certainly it is pure paranoia, but if we load TIME-WAIT
 *   with this semantics, we MUST NOT kill TIME-WAIT state with RSTs.
 * * If we invented some more clever way to catch duplicates
 *   (f.e. based on PAWS), we could truncate TIME-WAIT to several RTOs.
 *
 * The algorithm below is based on FORMAL INTERPRETATION of RFCs.
 * When you compare it to RFCs, please, read section SEGMENT ARRIVES
 * from the very beginning.
 *
 * NOTE. With recycling (and later with fin-wait-2) TW bucket
 * is _not_ stateless. It means, that strictly speaking we must
 * spinlock it. I do not want! Well, probability of misbehaviour
 * is ridiculously low and, seems, we could use some mb() tricks
 * to avoid misread sequence numbers, states etc.  --ANK
 *
 * We don't need to initialize tmp_out.sack_ok as we don't use the results
 */
enum tcp_tw_status
tcp_timewait_state_process(struct inet_timewait_sock *tw, struct sk_buff *skb,
                           const struct tcphdr *th, u32 *tw_isn,
                           enum skb_drop_reason *drop_reason)
{
        struct tcp_timewait_sock *tcptw = tcp_twsk((struct sock *)tw);
        u32 rcv_nxt = READ_ONCE(tcptw->tw_rcv_nxt);
        struct tcp_options_received tmp_opt;
        enum skb_drop_reason psp_drop;
        bool paws_reject = false;
        int ts_recent_stamp;

        /* Instead of dropping immediately, wait to see what value is
         * returned. We will accept a non psp-encapsulated syn in the
         * case where TCP_TW_SYN is returned.
         */
        psp_drop = psp_twsk_rx_policy_check(tw, skb);

        tmp_opt.saw_tstamp = 0;
        ts_recent_stamp = READ_ONCE(tcptw->tw_ts_recent_stamp);
        if (th->doff > (sizeof(*th) >> 2) && ts_recent_stamp) {
                tcp_parse_options(twsk_net(tw), skb, &tmp_opt, 0, NULL);

                if (tmp_opt.saw_tstamp) {
                        if (tmp_opt.rcv_tsecr)
                                tmp_opt.rcv_tsecr -= tcptw->tw_ts_offset;
                        tmp_opt.ts_recent        = READ_ONCE(tcptw->tw_ts_recent);
                        tmp_opt.ts_recent_stamp        = ts_recent_stamp;
                        paws_reject = tcp_paws_reject(&tmp_opt, th->rst);
                }
        }

        if (READ_ONCE(tw->tw_substate) == TCP_FIN_WAIT2) {
                /* Just repeat all the checks of tcp_rcv_state_process() */

                if (psp_drop)
                        goto out_put;

                /* Out of window, send ACK */
                if (paws_reject ||
                    !tcp_in_window(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq,
                                   rcv_nxt,
                                   rcv_nxt + tcptw->tw_rcv_wnd))
                        return tcp_timewait_check_oow_rate_limit(
                                tw, skb, LINUX_MIB_TCPACKSKIPPEDFINWAIT2);

                if (th->rst)
                        goto kill;

                if (th->syn && !before(TCP_SKB_CB(skb)->seq, rcv_nxt))
                        return TCP_TW_RST;

                /* Dup ACK? */
                if (!th->ack ||
                    !after(TCP_SKB_CB(skb)->end_seq, rcv_nxt) ||
                    TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq) {
                        inet_twsk_put(tw);
                        return TCP_TW_SUCCESS;
                }

                /* New data or FIN. If new data arrive after half-duplex close,
                 * reset.
                 */
                if (!th->fin ||
                    TCP_SKB_CB(skb)->end_seq != rcv_nxt + 1)
                        return TCP_TW_RST;

                /* FIN arrived, enter true time-wait state. */
                WRITE_ONCE(tw->tw_substate, TCP_TIME_WAIT);
                twsk_rcv_nxt_update(tcptw, TCP_SKB_CB(skb)->end_seq,
                                    rcv_nxt);

                if (tmp_opt.saw_tstamp) {
                        u64 ts = tcp_clock_ms();

                        WRITE_ONCE(tw->tw_entry_stamp, ts);
                        WRITE_ONCE(tcptw->tw_ts_recent_stamp,
                                   div_u64(ts, MSEC_PER_SEC));
                        WRITE_ONCE(tcptw->tw_ts_recent,
                                   tmp_opt.rcv_tsval);
                }

                inet_twsk_reschedule(tw, TCP_TIMEWAIT_LEN);
                return TCP_TW_ACK;
        }

        /*
         *        Now real TIME-WAIT state.
         *
         *        RFC 1122:
         *        "When a connection is [...] on TIME-WAIT state [...]
         *        [a TCP] MAY accept a new SYN from the remote TCP to
         *        reopen the connection directly, if it:
         *
         *        (1)  assigns its initial sequence number for the new
         *        connection to be larger than the largest sequence
         *        number it used on the previous connection incarnation,
         *        and
         *
         *        (2)  returns to TIME-WAIT state if the SYN turns out
         *        to be an old duplicate".
         */

        if (!paws_reject &&
            (TCP_SKB_CB(skb)->seq == rcv_nxt &&
             (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq || th->rst))) {
                /* In window segment, it may be only reset or bare ack. */

                if (psp_drop)
                        goto out_put;

                if (th->rst) {
                        /* This is TIME_WAIT assassination, in two flavors.
                         * Oh well... nobody has a sufficient solution to this
                         * protocol bug yet.
                         */
                        if (!READ_ONCE(twsk_net(tw)->ipv4.sysctl_tcp_rfc1337)) {
kill:
                                inet_twsk_deschedule_put(tw);
                                return TCP_TW_SUCCESS;
                        }
                } else {
                        inet_twsk_reschedule(tw, TCP_TIMEWAIT_LEN);
                }

                if (tmp_opt.saw_tstamp) {
                        WRITE_ONCE(tcptw->tw_ts_recent,
                                   tmp_opt.rcv_tsval);
                        WRITE_ONCE(tcptw->tw_ts_recent_stamp,
                                   ktime_get_seconds());
                }

                inet_twsk_put(tw);
                return TCP_TW_SUCCESS;
        }

        /* Out of window segment.

           All the segments are ACKed immediately.

           The only exception is new SYN. We accept it, if it is
           not old duplicate and we are not in danger to be killed
           by delayed old duplicates. RFC check is that it has
           newer sequence number works at rates <40Mbit/sec.
           However, if paws works, it is reliable AND even more,
           we even may relax silly seq space cutoff.

           RED-PEN: we violate main RFC requirement, if this SYN will appear
           old duplicate (i.e. we receive RST in reply to SYN-ACK),
           we must return socket to time-wait state. It is not good,
           but not fatal yet.
         */

        if (th->syn && !th->rst && !th->ack && !paws_reject &&
            (after(TCP_SKB_CB(skb)->seq, rcv_nxt) ||
             (tmp_opt.saw_tstamp &&
              (s32)(READ_ONCE(tcptw->tw_ts_recent) - tmp_opt.rcv_tsval) < 0))) {
                u32 isn = tcptw->tw_snd_nxt + 65535 + 2;
                if (isn == 0)
                        isn++;
                *tw_isn = isn;
                return TCP_TW_SYN;
        }

        if (psp_drop)
                goto out_put;

        if (paws_reject) {
                *drop_reason = SKB_DROP_REASON_TCP_RFC7323_TW_PAWS;
                __NET_INC_STATS(twsk_net(tw), LINUX_MIB_PAWS_TW_REJECTED);
        }

        if (!th->rst) {
                /* In this case we must reset the TIMEWAIT timer.
                 *
                 * If it is ACKless SYN it may be both old duplicate
                 * and new good SYN with random sequence number <rcv_nxt.
                 * Do not reschedule in the last case.
                 */
                if (paws_reject || th->ack)
                        inet_twsk_reschedule(tw, TCP_TIMEWAIT_LEN);

                return tcp_timewait_check_oow_rate_limit(
                        tw, skb, LINUX_MIB_TCPACKSKIPPEDTIMEWAIT);
        }

out_put:
        inet_twsk_put(tw);
        return TCP_TW_SUCCESS;
}

static void tcp_time_wait_init(struct sock *sk, struct tcp_timewait_sock *tcptw)
{
#ifdef CONFIG_TCP_MD5SIG
        const struct tcp_sock *tp = tcp_sk(sk);
        struct tcp_md5sig_key *key;

        /*
         * The timewait bucket does not have the key DB from the
         * sock structure. We just make a quick copy of the
         * md5 key being used (if indeed we are using one)
         * so the timewait ack generating code has the key.
         */
        tcptw->tw_md5_key = NULL;
        if (!static_branch_unlikely(&tcp_md5_needed.key))
                return;

        key = tp->af_specific->md5_lookup(sk, sk);
        if (key) {
                tcptw->tw_md5_key = kmemdup(key, sizeof(*key), GFP_ATOMIC);
                if (!tcptw->tw_md5_key)
                        return;
                if (!static_key_fast_inc_not_disabled(&tcp_md5_needed.key.key))
                        goto out_free;
        }
        return;
out_free:
        WARN_ON_ONCE(1);
        kfree(tcptw->tw_md5_key);
        tcptw->tw_md5_key = NULL;
#endif
}

/*
 * Move a socket to time-wait or dead fin-wait-2 state.
 */
void tcp_time_wait(struct sock *sk, int state, int timeo)
{
        const struct inet_connection_sock *icsk = inet_csk(sk);
        struct tcp_sock *tp = tcp_sk(sk);
        struct net *net = sock_net(sk);
        struct inet_timewait_sock *tw;

        tw = inet_twsk_alloc(sk, &net->ipv4.tcp_death_row, state);

        if (tw) {
                struct tcp_timewait_sock *tcptw = tcp_twsk((struct sock *)tw);
                const int rto = (icsk->icsk_rto << 2) - (icsk->icsk_rto >> 1);

                tw->tw_mark                = sk->sk_mark;
                tw->tw_priority                = READ_ONCE(sk->sk_priority);
                tw->tw_rcv_wscale        = tp->rx_opt.rcv_wscale;
                /* refreshed when we enter true TIME-WAIT state */
                tw->tw_entry_stamp        = tcp_time_stamp_ms(tp);
                tcptw->tw_rcv_nxt        = tp->rcv_nxt;
                tcptw->tw_snd_nxt        = tp->snd_nxt;
                tcptw->tw_rcv_wnd        = tcp_receive_window(tp);
                tcptw->tw_ts_recent        = tp->rx_opt.ts_recent;
                tcptw->tw_ts_recent_stamp = tp->rx_opt.ts_recent_stamp;
                tcptw->tw_ts_offset        = tp->tsoffset;
                tw->tw_usec_ts                = tp->tcp_usec_ts;
                tcptw->tw_last_oow_ack_time = 0;
                tcptw->tw_tx_delay        = tp->tcp_tx_delay;
                tw->tw_txhash                = sk->sk_txhash;
                tw->tw_tx_queue_mapping = sk->sk_tx_queue_mapping;
#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
                tw->tw_rx_queue_mapping = sk->sk_rx_queue_mapping;
#endif
#if IS_ENABLED(CONFIG_IPV6)
                if (tw->tw_family == PF_INET6) {
                        struct ipv6_pinfo *np = inet6_sk(sk);

                        tw->tw_v6_daddr = sk->sk_v6_daddr;
                        tw->tw_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
                        tw->tw_tclass = np->tclass;
                        tw->tw_flowlabel = be32_to_cpu(np->flow_label & IPV6_FLOWLABEL_MASK);
                        tw->tw_ipv6only = sk->sk_ipv6only;
                }
#endif

                tcp_time_wait_init(sk, tcptw);
                tcp_ao_time_wait(tcptw, tp);

                /* Get the TIME_WAIT timeout firing. */
                if (timeo < rto)
                        timeo = rto;

                if (state == TCP_TIME_WAIT)
                        timeo = TCP_TIMEWAIT_LEN;

                /* Linkage updates.
                 * Note that access to tw after this point is illegal.
                 */
                inet_twsk_hashdance_schedule(tw, sk, net->ipv4.tcp_death_row.hashinfo, timeo);
        } else {
                /* Sorry, if we're out of memory, just CLOSE this
                 * socket up.  We've got bigger problems than
                 * non-graceful socket closings.
                 */
                NET_INC_STATS(net, LINUX_MIB_TCPTIMEWAITOVERFLOW);
        }

        tcp_update_metrics(sk);
        tcp_done(sk);
}

void tcp_twsk_destructor(struct sock *sk)
{
#ifdef CONFIG_TCP_MD5SIG
        if (static_branch_unlikely(&tcp_md5_needed.key)) {
                struct tcp_timewait_sock *twsk = tcp_twsk(sk);

                if (twsk->tw_md5_key) {
                        kfree(twsk->tw_md5_key);
                        static_branch_slow_dec_deferred(&tcp_md5_needed);
                }
        }
#endif
        tcp_ao_destroy_sock(sk, true);
        psp_twsk_assoc_free(inet_twsk(sk));
}

void tcp_twsk_purge(struct list_head *net_exit_list)
{
        bool purged_once = false;
        struct net *net;

        list_for_each_entry(net, net_exit_list, exit_list) {
                if (net->ipv4.tcp_death_row.hashinfo->pernet) {
                        /* Even if tw_refcount == 1, we must clean up kernel reqsk */
                        inet_twsk_purge(net->ipv4.tcp_death_row.hashinfo);
                } else if (!purged_once) {
                        inet_twsk_purge(&tcp_hashinfo);
                        purged_once = true;
                }
        }
}

/* Warning : This function is called without sk_listener being locked.
 * Be sure to read socket fields once, as their value could change under us.
 */
void tcp_openreq_init_rwin(struct request_sock *req,
                           const struct sock *sk_listener,
                           const struct dst_entry *dst)
{
        struct inet_request_sock *ireq = inet_rsk(req);
        const struct tcp_sock *tp = tcp_sk(sk_listener);
        int full_space = tcp_full_space(sk_listener);
        u32 window_clamp;
        __u8 rcv_wscale;
        u32 rcv_wnd;
        int mss;

        mss = tcp_mss_clamp(tp, tcp_dst_advmss(dst));
        window_clamp = READ_ONCE(tp->window_clamp);
        /* Set this up on the first call only */
        req->rsk_window_clamp = window_clamp ? : dst_metric(dst, RTAX_WINDOW);

        /* limit the window selection if the user enforce a smaller rx buffer */
        if (sk_listener->sk_userlocks & SOCK_RCVBUF_LOCK &&
            (req->rsk_window_clamp > full_space || req->rsk_window_clamp == 0))
                req->rsk_window_clamp = full_space;

        rcv_wnd = tcp_rwnd_init_bpf((struct sock *)req);
        if (rcv_wnd == 0)
                rcv_wnd = dst_metric(dst, RTAX_INITRWND);
        else if (full_space < (u64)rcv_wnd * mss)
                full_space = min_t(u64, (u64)rcv_wnd * mss, INT_MAX);

        /* tcp_full_space because it is guaranteed to be the first packet */
        tcp_select_initial_window(sk_listener, full_space,
                mss - (ireq->tstamp_ok ? TCPOLEN_TSTAMP_ALIGNED : 0),
                &req->rsk_rcv_wnd,
                &req->rsk_window_clamp,
                ireq->wscale_ok,
                &rcv_wscale,
                rcv_wnd);
        ireq->rcv_wscale = rcv_wscale;
}

static void tcp_ecn_openreq_child(struct sock *sk,
                                  const struct request_sock *req,
                                  const struct sk_buff *skb)
{
        const struct tcp_request_sock *treq = tcp_rsk(req);
        struct tcp_sock *tp = tcp_sk(sk);

        if (treq->accecn_ok) {
                tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN);
                tp->syn_ect_snt = treq->syn_ect_snt;
                tcp_accecn_third_ack(sk, skb, treq->syn_ect_snt);
                tp->saw_accecn_opt = treq->saw_accecn_opt;
                if (treq->accecn_fail_mode & TCP_ACCECN_ACE_FAIL_SEND)
                        tcp_accecn_fail_mode_set(tp, TCP_ACCECN_ACE_FAIL_SEND);
                if (treq->accecn_fail_mode & TCP_ACCECN_ACE_FAIL_RECV)
                        tcp_accecn_fail_mode_set(tp, TCP_ACCECN_ACE_FAIL_RECV);
                tp->prev_ecnfield = treq->syn_ect_rcv;
                tp->accecn_opt_demand = 1;
                tcp_ecn_received_counters_payload(sk, skb);
        } else {
                if (inet_rsk(req)->ecn_ok && !tcp_ca_no_fallback_rfc3168(sk))
                        tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168);
                else
                        tcp_ecn_mode_set(tp, TCP_ECN_DISABLED);
        }
}

void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst)
{
        struct inet_connection_sock *icsk = inet_csk(sk);
        u32 ca_key = dst_metric(dst, RTAX_CC_ALGO);
        bool ca_got_dst = false;

        if (ca_key != TCP_CA_UNSPEC) {
                const struct tcp_congestion_ops *ca;

                rcu_read_lock();
                ca = tcp_ca_find_key(ca_key);
                if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
                        icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
                        WRITE_ONCE(icsk->icsk_ca_ops, ca);
                        ca_got_dst = true;
                }
                rcu_read_unlock();
        }

        /* If no valid choice made yet, assign current system default ca. */
        if (!ca_got_dst &&
            (!icsk->icsk_ca_setsockopt ||
             !bpf_try_module_get(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner)))
                tcp_assign_congestion_control(sk);

        tcp_set_ca_state(sk, TCP_CA_Open);
}

static void smc_check_reset_syn_req(const struct tcp_sock *oldtp,
                                    struct request_sock *req,
                                    struct tcp_sock *newtp)
{
#if IS_ENABLED(CONFIG_SMC)
        struct inet_request_sock *ireq;

        if (static_branch_unlikely(&tcp_have_smc)) {
                ireq = inet_rsk(req);
                if (oldtp->syn_smc && !ireq->smc_ok)
                        newtp->syn_smc = 0;
        }
#endif
}

/* This is not only more efficient than what we used to do, it eliminates
 * a lot of code duplication between IPv4/IPv6 SYN recv processing. -DaveM
 *
 * Actually, we could lots of memory writes here. tp of listening
 * socket contains all necessary default parameters.
 */
struct sock *tcp_create_openreq_child(const struct sock *sk,
                                      struct request_sock *req,
                                      struct sk_buff *skb)
{
        struct sock *newsk = inet_csk_clone_lock(sk, req, GFP_ATOMIC);
        const struct inet_request_sock *ireq = inet_rsk(req);
        struct tcp_request_sock *treq = tcp_rsk(req);
        struct inet_connection_sock *newicsk;
        const struct tcp_sock *oldtp;
        struct tcp_sock *newtp;
        u32 seq;

        if (!newsk)
                return NULL;

        newicsk = inet_csk(newsk);
        newtp = tcp_sk(newsk);
        oldtp = tcp_sk(sk);

        smc_check_reset_syn_req(oldtp, req, newtp);

        /* Now setup tcp_sock */
        newtp->pred_flags = 0;

        seq = treq->rcv_isn + 1;
        newtp->rcv_wup = seq;
        WRITE_ONCE(newtp->copied_seq, seq);
        WRITE_ONCE(newtp->rcv_nxt, seq);
        newtp->segs_in = 1;

        seq = treq->snt_isn + 1;
        newtp->snd_sml = newtp->snd_una = seq;
        WRITE_ONCE(newtp->snd_nxt, seq);
        newtp->snd_up = seq;

        INIT_LIST_HEAD(&newtp->tsq_node);
        INIT_LIST_HEAD(&newtp->tsorted_sent_queue);

        tcp_init_wl(newtp, treq->rcv_isn);

        minmax_reset(&newtp->rtt_min, tcp_jiffies32, ~0U);
        newicsk->icsk_ack.lrcvtime = tcp_jiffies32;

        newtp->lsndtime = tcp_jiffies32;
        newsk->sk_txhash = READ_ONCE(treq->txhash);
        newtp->total_retrans = req->num_retrans;

        tcp_init_xmit_timers(newsk);
        WRITE_ONCE(newtp->write_seq, newtp->pushed_seq = treq->snt_isn + 1);

        if (sock_flag(newsk, SOCK_KEEPOPEN))
                tcp_reset_keepalive_timer(newsk, keepalive_time_when(newtp));

        newtp->rx_opt.tstamp_ok = ireq->tstamp_ok;
        newtp->rx_opt.sack_ok = ireq->sack_ok;
        newtp->window_clamp = req->rsk_window_clamp;
        newtp->rcv_ssthresh = req->rsk_rcv_wnd;
        newtp->rcv_wnd = req->rsk_rcv_wnd;
        newtp->rcv_mwnd_seq = newtp->rcv_wup + req->rsk_rcv_wnd;
        newtp->rx_opt.wscale_ok = ireq->wscale_ok;
        if (newtp->rx_opt.wscale_ok) {
                newtp->rx_opt.snd_wscale = ireq->snd_wscale;
                newtp->rx_opt.rcv_wscale = ireq->rcv_wscale;
        } else {
                newtp->rx_opt.snd_wscale = newtp->rx_opt.rcv_wscale = 0;
                newtp->window_clamp = min(newtp->window_clamp, 65535U);
        }
        newtp->snd_wnd = ntohs(tcp_hdr(skb)->window) << newtp->rx_opt.snd_wscale;
        newtp->max_window = newtp->snd_wnd;

        if (newtp->rx_opt.tstamp_ok) {
                newtp->tcp_usec_ts = treq->req_usec_ts;
                newtp->rx_opt.ts_recent = req->ts_recent;
                newtp->rx_opt.ts_recent_stamp = ktime_get_seconds();
                newtp->tcp_header_len = sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
        } else {
                newtp->tcp_usec_ts = 0;
                newtp->rx_opt.ts_recent_stamp = 0;
                newtp->tcp_header_len = sizeof(struct tcphdr);
        }
        if (req->num_timeout) {
                newtp->total_rto = req->num_timeout;
                newtp->undo_marker = treq->snt_isn;
                if (newtp->tcp_usec_ts) {
                        newtp->retrans_stamp = treq->snt_synack;
                        newtp->total_rto_time = (u32)(tcp_clock_us() -
                                                      newtp->retrans_stamp) / USEC_PER_MSEC;
                } else {
                        newtp->retrans_stamp = div_u64(treq->snt_synack,
                                                       USEC_PER_SEC / TCP_TS_HZ);
                        newtp->total_rto_time = tcp_clock_ms() -
                                                newtp->retrans_stamp;
                }
                newtp->total_rto_recoveries = 1;
        }
        newtp->tsoffset = treq->ts_off;
#ifdef CONFIG_TCP_MD5SIG
        newtp->md5sig_info = NULL;        /*XXX*/
#endif
#ifdef CONFIG_TCP_AO
        newtp->ao_info = NULL;

        if (tcp_rsk_used_ao(req)) {
                struct tcp_ao_key *ao_key;

                ao_key = treq->af_specific->ao_lookup(sk, req, tcp_rsk(req)->ao_keyid, -1);
                if (ao_key)
                        newtp->tcp_header_len += tcp_ao_len_aligned(ao_key);
        }
 #endif
        if (skb->len >= TCP_MSS_DEFAULT + newtp->tcp_header_len)
                newicsk->icsk_ack.last_seg_size = skb->len - newtp->tcp_header_len;
        newtp->rx_opt.mss_clamp = req->mss;
        tcp_ecn_openreq_child(newsk, req, skb);
        newtp->fastopen_req = NULL;
        RCU_INIT_POINTER(newtp->fastopen_rsk, NULL);

        newtp->bpf_chg_cc_inprogress = 0;
        tcp_bpf_clone(sk, newsk);

        __TCP_INC_STATS(sock_net(sk), TCP_MIB_PASSIVEOPENS);

        xa_init_flags(&newsk->sk_user_frags, XA_FLAGS_ALLOC1);

        return newsk;
}

/*
 * Process an incoming packet for SYN_RECV sockets represented as a
 * request_sock. Normally sk is the listener socket but for TFO it
 * points to the child socket.
 *
 * XXX (TFO) - The current impl contains a special check for ack
 * validation and inside tcp_v4_reqsk_send_ack(). Can we do better?
 *
 * We don't need to initialize tmp_opt.sack_ok as we don't use the results
 *
 * Note: If @fastopen is true, this can be called from process context.
 *       Otherwise, this is from BH context.
 */

struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
                           struct request_sock *req,
                           bool fastopen, bool *req_stolen,
                           enum skb_drop_reason *drop_reason)
{
        struct tcp_options_received tmp_opt;
        struct sock *child;
        const struct tcphdr *th = tcp_hdr(skb);
        __be32 flg = tcp_flag_word(th) & (TCP_FLAG_RST|TCP_FLAG_SYN|TCP_FLAG_ACK);
        bool tsecr_reject = false;
        bool paws_reject = false;
        bool own_req;

        tmp_opt.saw_tstamp = 0;
        tmp_opt.accecn = 0;
        if (th->doff > (sizeof(struct tcphdr)>>2)) {
                tcp_parse_options(sock_net(sk), skb, &tmp_opt, 0, NULL);

                if (tmp_opt.saw_tstamp) {
                        tmp_opt.ts_recent = req->ts_recent;
                        if (tmp_opt.rcv_tsecr) {
                                if (inet_rsk(req)->tstamp_ok && !fastopen)
                                        tsecr_reject = !between(tmp_opt.rcv_tsecr,
                                                        tcp_rsk(req)->snt_tsval_first,
                                                        READ_ONCE(tcp_rsk(req)->snt_tsval_last));
                                tmp_opt.rcv_tsecr -= tcp_rsk(req)->ts_off;
                        }
                        /* We do not store true stamp, but it is not required,
                         * it can be estimated (approximately)
                         * from another data.
                         */
                        tmp_opt.ts_recent_stamp = ktime_get_seconds() -
                                tcp_reqsk_timeout(req) / HZ;
                        paws_reject = tcp_paws_reject(&tmp_opt, th->rst);
                }
        }

        /* Check for pure retransmitted SYN. */
        if (TCP_SKB_CB(skb)->seq == tcp_rsk(req)->rcv_isn &&
            flg == TCP_FLAG_SYN &&
            !paws_reject) {
                /*
                 * RFC793 draws (Incorrectly! It was fixed in RFC1122)
                 * this case on figure 6 and figure 8, but formal
                 * protocol description says NOTHING.
                 * To be more exact, it says that we should send ACK,
                 * because this segment (at least, if it has no data)
                 * is out of window.
                 *
                 *  CONCLUSION: RFC793 (even with RFC1122) DOES NOT
                 *  describe SYN-RECV state. All the description
                 *  is wrong, we cannot believe to it and should
                 *  rely only on common sense and implementation
                 *  experience.
                 *
                 * Enforce "SYN-ACK" according to figure 8, figure 6
                 * of RFC793, fixed by RFC1122.
                 *
                 * Note that even if there is new data in the SYN packet
                 * they will be thrown away too.
                 *
                 * Reset timer after retransmitting SYNACK, similar to
                 * the idea of fast retransmit in recovery.
                 */
                if (!tcp_oow_rate_limited(sock_net(sk), skb,
                                          LINUX_MIB_TCPACKSKIPPEDSYNRECV,
                                          &tcp_rsk(req)->last_oow_ack_time)) {
                        if (tcp_rsk(req)->accecn_ok) {
                                u8 ect_rcv = TCP_SKB_CB(skb)->ip_dsfield &
                                             INET_ECN_MASK;

                                tcp_rsk(req)->syn_ect_rcv = ect_rcv;
                                if (tcp_accecn_ace(tcp_hdr(skb)) == 0x0)
                                        tcp_rsk(req)->accecn_fail_mode |= TCP_ACCECN_ACE_FAIL_RECV;
                        }
                        if (!tcp_rtx_synack(sk, req)) {
                                unsigned long expires = jiffies;

                                if (req->num_retrans > 1 && tcp_rsk(req)->accecn_ok)
                                        tcp_rsk(req)->accecn_fail_mode |= TCP_ACCECN_ACE_FAIL_SEND;

                                expires += tcp_reqsk_timeout(req);
                                if (!fastopen)
                                        mod_timer_pending(&req->rsk_timer,
                                                          expires);
                                else
                                        req->rsk_timer.expires = expires;
                        }
                }
                return NULL;
        }

        /* Further reproduces section "SEGMENT ARRIVES"
           for state SYN-RECEIVED of RFC793.
           It is broken, however, it does not work only
           when SYNs are crossed.

           You would think that SYN crossing is impossible here, since
           we should have a SYN_SENT socket (from connect()) on our end,
           but this is not true if the crossed SYNs were sent to both
           ends by a malicious third party.  We must defend against this,
           and to do that we first verify the ACK (as per RFC793, page
           36) and reset if it is invalid.  Is this a true full defense?
           To convince ourselves, let us consider a way in which the ACK
           test can still pass in this 'malicious crossed SYNs' case.
           Malicious sender sends identical SYNs (and thus identical sequence
           numbers) to both A and B:

                A: gets SYN, seq=7
                B: gets SYN, seq=7

           By our good fortune, both A and B select the same initial
           send sequence number of seven :-)

                A: sends SYN|ACK, seq=7, ack_seq=8
                B: sends SYN|ACK, seq=7, ack_seq=8

           So we are now A eating this SYN|ACK, ACK test passes.  So
           does sequence test, SYN is truncated, and thus we consider
           it a bare ACK.

           If icsk->icsk_accept_queue.rskq_defer_accept, we silently drop this
           bare ACK.  Otherwise, we create an established connection.  Both
           ends (listening sockets) accept the new incoming connection and try
           to talk to each other. 8-)

           Note: This case is both harmless, and rare.  Possibility is about the
           same as us discovering intelligent life on another plant tomorrow.

           But generally, we should (RFC lies!) to accept ACK
           from SYNACK both here and in tcp_rcv_state_process().
           tcp_rcv_state_process() does not, hence, we do not too.

           Note that the case is absolutely generic:
           we cannot optimize anything here without
           violating protocol. All the checks must be made
           before attempt to create socket.
         */

        /* RFC793 page 36: "If the connection is in any non-synchronized state ...
         *                  and the incoming segment acknowledges something not yet
         *                  sent (the segment carries an unacceptable ACK) ...
         *                  a reset is sent."
         *
         * Invalid ACK: reset will be sent by listening socket.
         * Note that the ACK validity check for a Fast Open socket is done
         * elsewhere and is checked directly against the child socket rather
         * than req because user data may have been sent out.
         */
        if ((flg & TCP_FLAG_ACK) && !(flg & TCP_FLAG_RST) && !fastopen &&
            (TCP_SKB_CB(skb)->ack_seq !=
             tcp_rsk(req)->snt_isn + 1))
                return sk;

        /* RFC793: "first check sequence number". */

        if (paws_reject || tsecr_reject ||
            !tcp_in_window(TCP_SKB_CB(skb)->seq,
                           TCP_SKB_CB(skb)->end_seq,
                           tcp_rsk(req)->rcv_nxt,
                           tcp_rsk(req)->rcv_nxt +
                           tcp_synack_window(req))) {
                /* Out of window: send ACK and drop. */
                if (!(flg & TCP_FLAG_RST) &&
                    !tcp_oow_rate_limited(sock_net(sk), skb,
                                          LINUX_MIB_TCPACKSKIPPEDSYNRECV,
                                          &tcp_rsk(req)->last_oow_ack_time))
                        req->rsk_ops->send_ack(sk, skb, req);
                if (paws_reject) {
                        SKB_DR_SET(*drop_reason, TCP_RFC7323_PAWS);
                        NET_INC_STATS(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED);
                } else if (tsecr_reject) {
                        SKB_DR_SET(*drop_reason, TCP_RFC7323_TSECR);
                        NET_INC_STATS(sock_net(sk), LINUX_MIB_TSECRREJECTED);
                } else {
                        SKB_DR_SET(*drop_reason, TCP_OVERWINDOW);
                }
                return NULL;
        }

        /* In sequence, PAWS is OK. */

        if (TCP_SKB_CB(skb)->seq == tcp_rsk(req)->rcv_isn) {
                /* Truncate SYN, it is out of window starting
                   at tcp_rsk(req)->rcv_isn + 1. */
                flg &= ~TCP_FLAG_SYN;
        }

        /* RFC 5961 section 3.2, as clarified by RFC 9293 section
         * 3.10.7.4, requires a challenge ACK for a non-exact
         * in-window RST in SYN-RECEIVED.
         */
        if ((flg & TCP_FLAG_RST) &&
            TCP_SKB_CB(skb)->seq != tcp_rsk(req)->rcv_nxt) {
                tcp_reqsk_send_challenge_ack(sk, skb, req);
                return NULL;
        }

        /* RFC793: "second check the RST bit" and
         *           "fourth, check the SYN bit"
         */
        if (flg & (TCP_FLAG_RST|TCP_FLAG_SYN)) {
                TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
                goto embryonic_reset;
        }

        /* ACK sequence verified above, just make sure ACK is
         * set.  If ACK not set, just silently drop the packet.
         *
         * XXX (TFO) - if we ever allow "data after SYN", the
         * following check needs to be removed.
         */
        if (!(flg & TCP_FLAG_ACK))
                return NULL;

        if (tcp_rsk(req)->accecn_ok && tmp_opt.accecn &&
            tcp_rsk(req)->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) {
                u8 saw_opt = tcp_accecn_option_init(skb, tmp_opt.accecn);

                tcp_rsk(req)->saw_accecn_opt = saw_opt;
                if (tcp_rsk(req)->saw_accecn_opt == TCP_ACCECN_OPT_FAIL_SEEN) {
                        u8 fail_mode = TCP_ACCECN_OPT_FAIL_RECV;

                        tcp_rsk(req)->accecn_fail_mode |= fail_mode;
                }
        }

        /* For Fast Open no more processing is needed (sk is the
         * child socket).
         */
        if (fastopen)
                return sk;

        /* While TCP_DEFER_ACCEPT is active, drop bare ACK. */
        if (req->num_timeout < READ_ONCE(inet_csk(sk)->icsk_accept_queue.rskq_defer_accept) &&
            TCP_SKB_CB(skb)->end_seq == tcp_rsk(req)->rcv_isn + 1) {
                inet_rsk(req)->acked = 1;
                __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDEFERACCEPTDROP);
                return NULL;
        }

        /* OK, ACK is valid, create big socket and
         * feed this segment to it. It will repeat all
         * the tests. THIS SEGMENT MUST MOVE SOCKET TO
         * ESTABLISHED STATE. If it will be dropped after
         * socket is created, wait for troubles.
         */
        child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL,
                                                         req, &own_req, NULL);
        if (!child)
                goto listen_overflow;

        if (own_req && tmp_opt.saw_tstamp &&
            !after(TCP_SKB_CB(skb)->seq, tcp_rsk(req)->rcv_nxt))
                tcp_sk(child)->rx_opt.ts_recent = tmp_opt.rcv_tsval;

        if (own_req && rsk_drop_req(req)) {
                reqsk_queue_removed(&inet_csk(req->rsk_listener)->icsk_accept_queue, req);
                inet_csk_reqsk_queue_drop_and_put(req->rsk_listener, req);
                return child;
        }

        sock_rps_save_rxhash(child, skb);
        tcp_synack_rtt_meas(child, req);
        *req_stolen = !own_req;
        return inet_csk_complete_hashdance(sk, child, req, own_req);

listen_overflow:
        SKB_DR_SET(*drop_reason, TCP_LISTEN_OVERFLOW);
        if (sk != req->rsk_listener)
                __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE);

        if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_abort_on_overflow)) {
                inet_rsk(req)->acked = 1;
                return NULL;
        }

embryonic_reset:
        if (!(flg & TCP_FLAG_RST)) {
                /* Received a bad SYN pkt - for TFO We try not to reset
                 * the local connection unless it's really necessary to
                 * avoid becoming vulnerable to outside attack aiming at
                 * resetting legit local connections.
                 */
                req->rsk_ops->send_reset(sk, skb, SK_RST_REASON_INVALID_SYN);
        } else if (fastopen) { /* received a valid RST pkt */
                reqsk_fastopen_remove(sk, req, true);
                tcp_reset(sk, skb);
        }
        if (!fastopen) {
                bool unlinked = inet_csk_reqsk_queue_drop(sk, req);

                if (unlinked)
                        __NET_INC_STATS(sock_net(sk), LINUX_MIB_EMBRYONICRSTS);
                *req_stolen = !unlinked;
        }
        return NULL;
}

/*
 * Queue segment on the new socket if the new socket is active,
 * otherwise we just shortcircuit this and continue with
 * the new socket.
 *
 * For the vast majority of cases child->sk_state will be TCP_SYN_RECV
 * when entering. But other states are possible due to a race condition
 * where after __inet_lookup_established() fails but before the listener
 * locked is obtained, other packets cause the same connection to
 * be created.
 */

enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child,
                                       struct sk_buff *skb)
        __releases(&((child)->sk_lock.slock))
{
        enum skb_drop_reason reason = SKB_NOT_DROPPED_YET;
        int state = child->sk_state;

        /* record sk_napi_id and sk_rx_queue_mapping of child. */
        sk_mark_napi_id_set(child, skb);

        tcp_segs_in(tcp_sk(child), skb);
        if (!sock_owned_by_user(child)) {
                reason = tcp_rcv_state_process(child, skb);
                /* Wakeup parent, send SIGIO */
                if (state == TCP_SYN_RECV && child->sk_state != state)
                        READ_ONCE(parent->sk_data_ready)(parent);
        } else {
                /* Alas, it is possible again, because we do lookup
                 * in main socket hash table and lock on listening
                 * socket does not protect us more.
                 */
                __sk_add_backlog(child, skb);
        }

        bh_unlock_sock(child);

        return reason;
}


























































  403 























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_CTYPE_H
#define _LINUX_CTYPE_H

#include <linux/compiler.h>

/*
 * NOTE! This ctype does not handle EOF like the standard C
 * library is required to.
 */

#define _U        0x01        /* upper */
#define _L        0x02        /* lower */
#define _D        0x04        /* digit */
#define _C        0x08        /* cntrl */
#define _P        0x10        /* punct */
#define _S        0x20        /* white space (space/lf/tab) */
#define _X        0x40        /* hex digit */
#define _SP        0x80        /* hard space (0x20) */

extern const unsigned char _ctype[];

#define __ismask(x) (_ctype[(int)(unsigned char)(x)])

#define isalnum(c)        ((__ismask(c)&(_U|_L|_D)) != 0)
#define isalpha(c)        ((__ismask(c)&(_U|_L)) != 0)
#define iscntrl(c)        ((__ismask(c)&(_C)) != 0)
#define isgraph(c)        ((__ismask(c)&(_P|_U|_L|_D)) != 0)
#define islower(c)        ((__ismask(c)&(_L)) != 0)
#define isprint(c)        ((__ismask(c)&(_P|_U|_L|_D|_SP)) != 0)
#define ispunct(c)        ((__ismask(c)&(_P)) != 0)
/* Note: isspace() must return false for %NUL-terminator */
#define isspace(c)        ((__ismask(c)&(_S)) != 0)
#define isupper(c)        ((__ismask(c)&(_U)) != 0)
#define isxdigit(c)        ((__ismask(c)&(_D|_X)) != 0)

#define isascii(c) (((unsigned char)(c))<=0x7f)
#define toascii(c) (((unsigned char)(c))&0x7f)

#if __has_builtin(__builtin_isdigit)
#define  isdigit(c) __builtin_isdigit(c)
#else
static inline int isdigit(int c)
{
        return '0' <= c && c <= '9';
}
#endif

static inline unsigned char __tolower(unsigned char c)
{
        if (isupper(c))
                c -= 'A'-'a';
        return c;
}

static inline unsigned char __toupper(unsigned char c)
{
        if (islower(c))
                c -= 'a'-'A';
        return c;
}

#define tolower(c) __tolower(c)
#define toupper(c) __toupper(c)

/*
 * Fast implementation of tolower() for internal usage. Do not use in your
 * code.
 */
static inline char _tolower(const char c)
{
        return c | 0x20;
}

/* Fast check for octal digit */
static inline int isodigit(const char c)
{
        return c >= '0' && c <= '7';
}

#endif


































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































 5552 














 5552 
 5552 







 1254 
 5542 














































































 5552 








 5552 















 5552 





 5552 









































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
// SPDX-License-Identifier: GPL-2.0-only
// Copyright (C) 2022 Linutronix GmbH, John Ogness
// Copyright (C) 2022 Intel, Thomas Gleixner

#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/console.h>
#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/irqflags.h>
#include <linux/kdb.h>
#include <linux/kthread.h>
#include <linux/minmax.h>
#include <linux/panic.h>
#include <linux/percpu.h>
#include <linux/preempt.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/stddef.h>
#include <linux/string.h>
#include <linux/types.h>
#include "internal.h"
#include "printk_ringbuffer.h"
/*
 * Printk console printing implementation for consoles which does not depend
 * on the legacy style console_lock mechanism.
 *
 * The state of the console is maintained in the "nbcon_state" atomic
 * variable.
 *
 * The console is locked when:
 *
 *   - The 'prio' field contains the priority of the context that owns the
 *     console. Only higher priority contexts are allowed to take over the
 *     lock. A value of 0 (NBCON_PRIO_NONE) means the console is not locked.
 *
 *   - The 'cpu' field denotes on which CPU the console is locked. It is used
 *     to prevent busy waiting on the same CPU. Also it informs the lock owner
 *     that it has lost the lock in a more complex scenario when the lock was
 *     taken over by a higher priority context, released, and taken on another
 *     CPU with the same priority as the interrupted owner.
 *
 * The acquire mechanism uses a few more fields:
 *
 *   - The 'req_prio' field is used by the handover approach to make the
 *     current owner aware that there is a context with a higher priority
 *     waiting for the friendly handover.
 *
 *   - The 'unsafe' field allows to take over the console in a safe way in the
 *     middle of emitting a message. The field is set only when accessing some
 *     shared resources or when the console device is manipulated. It can be
 *     cleared, for example, after emitting one character when the console
 *     device is in a consistent state.
 *
 *   - The 'unsafe_takeover' field is set when a hostile takeover took the
 *     console in an unsafe state. The console will stay in the unsafe state
 *     until re-initialized.
 *
 * The acquire mechanism uses three approaches:
 *
 *   1) Direct acquire when the console is not owned or is owned by a lower
 *      priority context and is in a safe state.
 *
 *   2) Friendly handover mechanism uses a request/grant handshake. It is used
 *      when the current owner has lower priority and the console is in an
 *      unsafe state.
 *
 *      The requesting context:
 *
 *        a) Sets its priority into the 'req_prio' field.
 *
 *        b) Waits (with a timeout) for the owning context to unlock the
 *           console.
 *
 *        c) Takes the lock and clears the 'req_prio' field.
 *
 *      The owning context:
 *
 *        a) Observes the 'req_prio' field set on exit from the unsafe
 *           console state.
 *
 *        b) Gives up console ownership by clearing the 'prio' field.
 *
 *   3) Unsafe hostile takeover allows to take over the lock even when the
 *      console is an unsafe state. It is used only in panic() by the final
 *      attempt to flush consoles in a try and hope mode.
 *
 *      Note that separate record buffers are used in panic(). As a result,
 *      the messages can be read and formatted without any risk even after
 *      using the hostile takeover in unsafe state.
 *
 * The release function simply clears the 'prio' field.
 *
 * All operations on @console::nbcon_state are atomic cmpxchg based to
 * handle concurrency.
 *
 * The acquire/release functions implement only minimal policies:
 *
 *   - Preference for higher priority contexts.
 *   - Protection of the panic CPU.
 *
 * All other policy decisions must be made at the call sites:
 *
 *   - What is marked as an unsafe section.
 *   - Whether to spin-wait if there is already an owner and the console is
 *     in an unsafe state.
 *   - Whether to attempt an unsafe hostile takeover.
 *
 * The design allows to implement the well known:
 *
 *     acquire()
 *     output_one_printk_record()
 *     release()
 *
 * The output of one printk record might be interrupted with a higher priority
 * context. The new owner is supposed to reprint the entire interrupted record
 * from scratch.
 */

/* Counter of active nbcon emergency contexts. */
static atomic_t nbcon_cpu_emergency_cnt = ATOMIC_INIT(0);

/**
 * nbcon_state_set - Helper function to set the console state
 * @con:        Console to update
 * @new:        The new state to write
 *
 * Only to be used when the console is not yet or no longer visible in the
 * system. Otherwise use nbcon_state_try_cmpxchg().
 */
static inline void nbcon_state_set(struct console *con, struct nbcon_state *new)
{
        atomic_set(&ACCESS_PRIVATE(con, nbcon_state), new->atom);
}

/**
 * nbcon_state_read - Helper function to read the console state
 * @con:        Console to read
 * @state:        The state to store the result
 */
static inline void nbcon_state_read(struct console *con, struct nbcon_state *state)
{
        state->atom = atomic_read(&ACCESS_PRIVATE(con, nbcon_state));
}

/**
 * nbcon_state_try_cmpxchg() - Helper function for atomic_try_cmpxchg() on console state
 * @con:        Console to update
 * @cur:        Old/expected state
 * @new:        New state
 *
 * Return: True on success. False on fail and @cur is updated.
 */
static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_state *cur,
                                           struct nbcon_state *new)
{
        return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom);
}

/**
 * nbcon_seq_read - Read the current console sequence
 * @con:        Console to read the sequence of
 *
 * Return:        Sequence number of the next record to print on @con.
 */
u64 nbcon_seq_read(struct console *con)
{
        unsigned long nbcon_seq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_seq));

        return __ulseq_to_u64seq(prb, nbcon_seq);
}

/**
 * nbcon_seq_force - Force console sequence to a specific value
 * @con:        Console to work on
 * @seq:        Sequence number value to set
 *
 * Only to be used during init (before registration) or in extreme situations
 * (such as panic with CONSOLE_REPLAY_ALL).
 */
void nbcon_seq_force(struct console *con, u64 seq)
{
        /*
         * If the specified record no longer exists, the oldest available record
         * is chosen. This is especially important on 32bit systems because only
         * the lower 32 bits of the sequence number are stored. The upper 32 bits
         * are derived from the sequence numbers available in the ringbuffer.
         */
        u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb));

        atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq));
}

/**
 * nbcon_seq_try_update - Try to update the console sequence number
 * @ctxt:        Pointer to an acquire context that contains
 *                all information about the acquire mode
 * @new_seq:        The new sequence number to set
 *
 * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to
 * the 64bit value). This could be a different value than @new_seq if
 * nbcon_seq_force() was used or the current context no longer owns the
 * console. In the later case, it will stop printing anyway.
 */
static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq)
{
        unsigned long nbcon_seq = __u64seq_to_ulseq(ctxt->seq);
        struct console *con = ctxt->console;

        if (atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_seq), &nbcon_seq,
                                    __u64seq_to_ulseq(new_seq))) {
                ctxt->seq = new_seq;
        } else {
                ctxt->seq = nbcon_seq_read(con);
        }
}

/**
 * nbcon_context_try_acquire_direct - Try to acquire directly
 * @ctxt:                The context of the caller
 * @cur:                The current console state
 * @is_reacquire:        This acquire is a reacquire
 *
 * Acquire the console when it is released. Also acquire the console when
 * the current owner has a lower priority and the console is in a safe state.
 *
 * Return:        0 on success. Otherwise, an error code on failure. Also @cur
 *                is updated to the latest state when failed to modify it.
 *
 * Errors:
 *
 *        -EPERM:                A panic is in progress and this is neither the panic
 *                        CPU nor is this a reacquire. Or the current owner or
 *                        waiter has the same or higher priority. No acquire
 *                        method can be successful in these cases.
 *
 *        -EBUSY:                The current owner has a lower priority but the console
 *                        in an unsafe state. The caller should try using
 *                        the handover acquire method.
 */
static int nbcon_context_try_acquire_direct(struct nbcon_context *ctxt,
                                            struct nbcon_state *cur, bool is_reacquire)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state new;

        do {
                /*
                 * Panic does not imply that the console is owned. However,
                 * since all non-panic CPUs are stopped during panic(), it
                 * is safer to have them avoid gaining console ownership.
                 *
                 * One exception is when kdb has locked for printing on this CPU.
                 *
                 * Second exception is a reacquire (and an unsafe takeover
                 * has not previously occurred) then it is allowed to attempt
                 * a direct acquire in panic. This gives console drivers an
                 * opportunity to perform any necessary cleanup if they were
                 * interrupted by the panic CPU while printing.
                 */
                if (panic_on_other_cpu() &&
                    !kdb_printf_on_this_cpu() &&
                    (!is_reacquire || cur->unsafe_takeover)) {
                        return -EPERM;
                }

                if (ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio)
                        return -EPERM;

                if (cur->unsafe)
                        return -EBUSY;

                /*
                 * The console should never be safe for a direct acquire
                 * if an unsafe hostile takeover has ever happened.
                 */
                WARN_ON_ONCE(cur->unsafe_takeover);

                new.atom = cur->atom;
                new.prio        = ctxt->prio;
                new.req_prio        = NBCON_PRIO_NONE;
                new.unsafe        = cur->unsafe_takeover;
                new.cpu                = cpu;

        } while (!nbcon_state_try_cmpxchg(con, cur, &new));

        return 0;
}

static bool nbcon_waiter_matches(struct nbcon_state *cur, int expected_prio)
{
        /*
         * The request context is well defined by the @req_prio because:
         *
         * - Only a context with a priority higher than the owner can become
         *   a waiter.
         * - Only a context with a priority higher than the waiter can
         *   directly take over the request.
         * - There are only three priorities.
         * - Only one CPU is allowed to request PANIC priority.
         * - Lower priorities are ignored during panic() until reboot.
         *
         * As a result, the following scenario is *not* possible:
         *
         * 1. This context is currently a waiter.
         * 2. Another context with a higher priority than this context
         *    directly takes ownership.
         * 3. The higher priority context releases the ownership.
         * 4. Another lower priority context takes the ownership.
         * 5. Another context with the same priority as this context
         *    creates a request and starts waiting.
         *
         * Event #1 implies this context is EMERGENCY.
         * Event #2 implies the new context is PANIC.
         * Event #3 occurs when panic() has flushed the console.
         * Event #4 occurs when a non-panic CPU reacquires.
         * Event #5 is not possible due to the panic_on_other_cpu() check
         *          in nbcon_context_try_acquire_handover().
         */

        return (cur->req_prio == expected_prio);
}

/**
 * nbcon_context_try_acquire_requested - Try to acquire after having
 *                                         requested a handover
 * @ctxt:        The context of the caller
 * @cur:        The current console state
 *
 * This is a helper function for nbcon_context_try_acquire_handover().
 * It is called when the console is in an unsafe state. The current
 * owner will release the console on exit from the unsafe region.
 *
 * Return:        0 on success and @cur is updated to the new console state.
 *                Otherwise an error code on failure.
 *
 * Errors:
 *
 *        -EPERM:                A panic is in progress and this is not the panic CPU
 *                        or this context is no longer the waiter.
 *
 *        -EBUSY:                The console is still locked. The caller should
 *                        continue waiting.
 *
 * Note: The caller must still remove the request when an error has occurred
 *       except when this context is no longer the waiter.
 */
static int nbcon_context_try_acquire_requested(struct nbcon_context *ctxt,
                                               struct nbcon_state *cur)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state new;

        /* Note that the caller must still remove the request! */
        if (panic_on_other_cpu())
                return -EPERM;

        /*
         * Note that the waiter will also change if there was an unsafe
         * hostile takeover.
         */
        if (!nbcon_waiter_matches(cur, ctxt->prio))
                return -EPERM;

        /* If still locked, caller should continue waiting. */
        if (cur->prio != NBCON_PRIO_NONE)
                return -EBUSY;

        /*
         * The previous owner should have never released ownership
         * in an unsafe region.
         */
        WARN_ON_ONCE(cur->unsafe);

        new.atom = cur->atom;
        new.prio        = ctxt->prio;
        new.req_prio        = NBCON_PRIO_NONE;
        new.unsafe        = cur->unsafe_takeover;
        new.cpu                = cpu;

        if (!nbcon_state_try_cmpxchg(con, cur, &new)) {
                /*
                 * The acquire could fail only when it has been taken
                 * over by a higher priority context.
                 */
                WARN_ON_ONCE(nbcon_waiter_matches(cur, ctxt->prio));
                return -EPERM;
        }

        /* Handover success. This context now owns the console. */
        return 0;
}

/**
 * nbcon_context_try_acquire_handover - Try to acquire via handover
 * @ctxt:        The context of the caller
 * @cur:        The current console state
 *
 * The function must be called only when the context has higher priority
 * than the current owner and the console is in an unsafe state.
 * It is the case when nbcon_context_try_acquire_direct() returns -EBUSY.
 *
 * The function sets "req_prio" field to make the current owner aware of
 * the request. Then it waits until the current owner releases the console,
 * or an even higher context takes over the request, or timeout expires.
 *
 * The current owner checks the "req_prio" field on exit from the unsafe
 * region and releases the console. It does not touch the "req_prio" field
 * so that the console stays reserved for the waiter.
 *
 * Return:        0 on success. Otherwise, an error code on failure. Also @cur
 *                is updated to the latest state when failed to modify it.
 *
 * Errors:
 *
 *        -EPERM:                A panic is in progress and this is not the panic CPU.
 *                        Or a higher priority context has taken over the
 *                        console or the handover request.
 *
 *        -EBUSY:                The current owner is on the same CPU so that the hand
 *                        shake could not work. Or the current owner is not
 *                        willing to wait (zero timeout). Or the console does
 *                        not enter the safe state before timeout passed. The
 *                        caller might still use the unsafe hostile takeover
 *                        when allowed.
 *
 *        -EAGAIN:        @cur has changed when creating the handover request.
 *                        The caller should retry with direct acquire.
 */
static int nbcon_context_try_acquire_handover(struct nbcon_context *ctxt,
                                              struct nbcon_state *cur)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state new;
        int timeout;
        int request_err = -EBUSY;

        /*
         * Check that the handover is called when the direct acquire failed
         * with -EBUSY.
         */
        WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);
        WARN_ON_ONCE(!cur->unsafe);

        /*
         * Panic does not imply that the console is owned. However, it
         * is critical that non-panic CPUs during panic are unable to
         * wait for a handover in order to satisfy the assumptions of
         * nbcon_waiter_matches(). In particular, the assumption that
         * lower priorities are ignored during panic.
         */
        if (panic_on_other_cpu())
                return -EPERM;

        /* Handover is not possible on the same CPU. */
        if (cur->cpu == cpu)
                return -EBUSY;

        /*
         * Console stays unsafe after an unsafe takeover until re-initialized.
         * Waiting is not going to help in this case.
         */
        if (cur->unsafe_takeover)
                return -EBUSY;

        /* Is the caller willing to wait? */
        if (ctxt->spinwait_max_us == 0)
                return -EBUSY;

        /*
         * Setup a request for the handover. The caller should try to acquire
         * the console directly when the current state has been modified.
         */
        new.atom = cur->atom;
        new.req_prio = ctxt->prio;
        if (!nbcon_state_try_cmpxchg(con, cur, &new))
                return -EAGAIN;

        cur->atom = new.atom;

        /* Wait until there is no owner and then acquire the console. */
        for (timeout = ctxt->spinwait_max_us; timeout >= 0; timeout--) {
                /* On successful acquire, this request is cleared. */
                request_err = nbcon_context_try_acquire_requested(ctxt, cur);
                if (!request_err)
                        return 0;

                /*
                 * If the acquire should be aborted, it must be ensured
                 * that the request is removed before returning to caller.
                 */
                if (request_err == -EPERM)
                        break;

                udelay(1);

                /* Re-read the state because some time has passed. */
                nbcon_state_read(con, cur);
        }

        /* Timed out or aborted. Carefully remove handover request. */
        do {
                /*
                 * No need to remove request if there is a new waiter. This
                 * can only happen if a higher priority context has taken over
                 * the console or the handover request.
                 */
                if (!nbcon_waiter_matches(cur, ctxt->prio))
                        return -EPERM;

                /* Unset request for handover. */
                new.atom = cur->atom;
                new.req_prio = NBCON_PRIO_NONE;
                if (nbcon_state_try_cmpxchg(con, cur, &new)) {
                        /*
                         * Request successfully unset. Report failure of
                         * acquiring via handover.
                         */
                        cur->atom = new.atom;
                        return request_err;
                }

                /*
                 * Unable to remove request. Try to acquire in case
                 * the owner has released the lock.
                 */
        } while (nbcon_context_try_acquire_requested(ctxt, cur));

        /* Lucky timing. The acquire succeeded while removing the request. */
        return 0;
}

/**
 * nbcon_context_try_acquire_hostile - Acquire via unsafe hostile takeover
 * @ctxt:        The context of the caller
 * @cur:        The current console state
 *
 * Acquire the console even in the unsafe state.
 *
 * It can be permitted by setting the 'allow_unsafe_takeover' field only
 * by the final attempt to flush messages in panic().
 *
 * Return:        0 on success. -EPERM when not allowed by the context.
 */
static int nbcon_context_try_acquire_hostile(struct nbcon_context *ctxt,
                                             struct nbcon_state *cur)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state new;

        if (!ctxt->allow_unsafe_takeover)
                return -EPERM;

        /* Ensure caller is allowed to perform unsafe hostile takeovers. */
        if (WARN_ON_ONCE(ctxt->prio != NBCON_PRIO_PANIC))
                return -EPERM;

        /*
         * Check that try_acquire_direct() and try_acquire_handover() returned
         * -EBUSY in the right situation.
         */
        WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);
        WARN_ON_ONCE(cur->unsafe != true);

        do {
                new.atom = cur->atom;
                new.cpu                        = cpu;
                new.prio                = ctxt->prio;
                new.unsafe                |= cur->unsafe_takeover;
                new.unsafe_takeover        |= cur->unsafe;

        } while (!nbcon_state_try_cmpxchg(con, cur, &new));

        return 0;
}

static struct printk_buffers panic_nbcon_pbufs;

/**
 * nbcon_context_try_acquire - Try to acquire nbcon console
 * @ctxt:                The context of the caller
 * @is_reacquire:        This acquire is a reacquire
 *
 * Context:        Under @ctxt->con->device_lock() or local_irq_save().
 * Return:        True if the console was acquired. False otherwise.
 *
 * If the caller allowed an unsafe hostile takeover, on success the
 * caller should check the current console state to see if it is
 * in an unsafe state. Otherwise, on success the caller may assume
 * the console is not in an unsafe state.
 */
static bool nbcon_context_try_acquire(struct nbcon_context *ctxt, bool is_reacquire)
{
        struct console *con = ctxt->console;
        struct nbcon_state cur;
        int err;

        nbcon_state_read(con, &cur);
try_again:
        err = nbcon_context_try_acquire_direct(ctxt, &cur, is_reacquire);
        if (err != -EBUSY)
                goto out;

        err = nbcon_context_try_acquire_handover(ctxt, &cur);
        if (err == -EAGAIN)
                goto try_again;
        if (err != -EBUSY)
                goto out;

        err = nbcon_context_try_acquire_hostile(ctxt, &cur);
out:
        if (err)
                return false;

        /* Acquire succeeded. */

        /* Assign the appropriate buffer for this context. */
        if (panic_on_this_cpu())
                ctxt->pbufs = &panic_nbcon_pbufs;
        else
                ctxt->pbufs = con->pbufs;

        /* Set the record sequence for this context to print. */
        ctxt->seq = nbcon_seq_read(ctxt->console);

        return true;
}

static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu,
                                int expected_prio)
{
        /*
         * A similar function, nbcon_waiter_matches(), only deals with
         * EMERGENCY and PANIC priorities. However, this function must also
         * deal with the NORMAL priority, which requires additional checks
         * and constraints.
         *
         * For the case where preemption and interrupts are disabled, it is
         * enough to also verify that the owning CPU has not changed.
         *
         * For the case where preemption or interrupts are enabled, an
         * external synchronization method *must* be used. In particular,
         * the driver-specific locking mechanism used in device_lock()
         * (including disabling migration) should be used. It prevents
         * scenarios such as:
         *
         * 1. [Task A] owns a context with NBCON_PRIO_NORMAL on [CPU X] and
         *    is scheduled out.
         * 2. Another context takes over the lock with NBCON_PRIO_EMERGENCY
         *    and releases it.
         * 3. [Task B] acquires a context with NBCON_PRIO_NORMAL on [CPU X]
         *    and is scheduled out.
         * 4. [Task A] gets running on [CPU X] and sees that the console is
         *    still owned by a task on [CPU X] with NBON_PRIO_NORMAL. Thus
         *    [Task A] thinks it is the owner when it is not.
         */

        if (cur->prio != expected_prio)
                return false;

        if (cur->cpu != expected_cpu)
                return false;

        return true;
}

/**
 * nbcon_context_release - Release the console
 * @ctxt:        The nbcon context from nbcon_context_try_acquire()
 */
static void nbcon_context_release(struct nbcon_context *ctxt)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state cur;
        struct nbcon_state new;

        nbcon_state_read(con, &cur);

        do {
                if (!nbcon_owner_matches(&cur, cpu, ctxt->prio))
                        break;

                new.atom = cur.atom;
                new.prio = NBCON_PRIO_NONE;

                /*
                 * If @unsafe_takeover is set, it is kept set so that
                 * the state remains permanently unsafe.
                 */
                new.unsafe |= cur.unsafe_takeover;

        } while (!nbcon_state_try_cmpxchg(con, &cur, &new));

        ctxt->pbufs = NULL;
}

/**
 * nbcon_context_can_proceed - Check whether ownership can proceed
 * @ctxt:        The nbcon context from nbcon_context_try_acquire()
 * @cur:        The current console state
 *
 * Return:        True if this context still owns the console. False if
 *                ownership was handed over or taken.
 *
 * Must be invoked when entering the unsafe state to make sure that it still
 * owns the lock. Also must be invoked when exiting the unsafe context
 * to eventually free the lock for a higher priority context which asked
 * for the friendly handover.
 *
 * It can be called inside an unsafe section when the console is just
 * temporary in safe state instead of exiting and entering the unsafe
 * state.
 *
 * Also it can be called in the safe context before doing an expensive
 * safe operation. It does not make sense to do the operation when
 * a higher priority context took the lock.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context.
 */
static bool nbcon_context_can_proceed(struct nbcon_context *ctxt, struct nbcon_state *cur)
{
        unsigned int cpu = smp_processor_id();

        /* Make sure this context still owns the console. */
        if (!nbcon_owner_matches(cur, cpu, ctxt->prio))
                return false;

        /* The console owner can proceed if there is no waiter. */
        if (cur->req_prio == NBCON_PRIO_NONE)
                return true;

        /*
         * A console owner within an unsafe region is always allowed to
         * proceed, even if there are waiters. It can perform a handover
         * when exiting the unsafe region. Otherwise the waiter will
         * need to perform an unsafe hostile takeover.
         */
        if (cur->unsafe)
                return true;

        /* Waiters always have higher priorities than owners. */
        WARN_ON_ONCE(cur->req_prio <= cur->prio);

        /*
         * Having a safe point for take over and eventually a few
         * duplicated characters or a full line is way better than a
         * hostile takeover. Post processing can take care of the garbage.
         * Release and hand over.
         */
        nbcon_context_release(ctxt);

        /*
         * It is not clear whether the waiter really took over ownership. The
         * outermost callsite must make the final decision whether console
         * ownership is needed for it to proceed. If yes, it must reacquire
         * ownership (possibly hostile) before carefully proceeding.
         *
         * The calling context no longer owns the console so go back all the
         * way instead of trying to implement reacquire heuristics in tons of
         * places.
         */
        return false;
}

/**
 * nbcon_can_proceed - Check whether ownership can proceed
 * @wctxt:        The write context that was handed to the write function
 *
 * Return:        True if this context still owns the console. False if
 *                ownership was handed over or taken.
 *
 * It is used in nbcon_enter_unsafe() to make sure that it still owns the
 * lock. Also it is used in nbcon_exit_unsafe() to eventually free the lock
 * for a higher priority context which asked for the friendly handover.
 *
 * It can be called inside an unsafe section when the console is just
 * temporary in safe state instead of exiting and entering the unsafe state.
 *
 * Also it can be called in the safe context before doing an expensive safe
 * operation. It does not make sense to do the operation when a higher
 * priority context took the lock.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context.
 */
bool nbcon_can_proceed(struct nbcon_write_context *wctxt)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        struct console *con = ctxt->console;
        struct nbcon_state cur;

        nbcon_state_read(con, &cur);

        return nbcon_context_can_proceed(ctxt, &cur);
}
EXPORT_SYMBOL_GPL(nbcon_can_proceed);

#define nbcon_context_enter_unsafe(c)        __nbcon_context_update_unsafe(c, true)
#define nbcon_context_exit_unsafe(c)        __nbcon_context_update_unsafe(c, false)

/**
 * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state
 * @ctxt:        The nbcon context from nbcon_context_try_acquire()
 * @unsafe:        The new value for the unsafe bit
 *
 * Return:        True if the unsafe state was updated and this context still
 *                owns the console. Otherwise false if ownership was handed
 *                over or taken.
 *
 * This function allows console owners to modify the unsafe status of the
 * console.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context.
 *
 * Internal helper to avoid duplicated code.
 */
static bool __nbcon_context_update_unsafe(struct nbcon_context *ctxt, bool unsafe)
{
        struct console *con = ctxt->console;
        struct nbcon_state cur;
        struct nbcon_state new;

        nbcon_state_read(con, &cur);

        do {
                /*
                 * The unsafe bit must not be cleared if an
                 * unsafe hostile takeover has occurred.
                 */
                if (!unsafe && cur.unsafe_takeover)
                        goto out;

                if (!nbcon_context_can_proceed(ctxt, &cur))
                        return false;

                new.atom = cur.atom;
                new.unsafe = unsafe;
        } while (!nbcon_state_try_cmpxchg(con, &cur, &new));

        cur.atom = new.atom;
out:
        return nbcon_context_can_proceed(ctxt, &cur);
}

void nbcon_write_context_set_buf(struct nbcon_write_context *wctxt,
                                 char *buf, unsigned int len)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        struct console *con = ctxt->console;
        struct nbcon_state cur;

        wctxt->outbuf = buf;
        wctxt->len = len;
        nbcon_state_read(con, &cur);
        wctxt->unsafe_takeover = cur.unsafe_takeover;
}

/**
 * nbcon_enter_unsafe - Enter an unsafe region in the driver
 * @wctxt:        The write context that was handed to the write function
 *
 * Return:        True if this context still owns the console. False if
 *                ownership was handed over or taken.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context.
 */
bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        bool is_owner;

        is_owner = nbcon_context_enter_unsafe(ctxt);
        if (!is_owner)
                nbcon_write_context_set_buf(wctxt, NULL, 0);
        return is_owner;
}
EXPORT_SYMBOL_GPL(nbcon_enter_unsafe);

/**
 * nbcon_exit_unsafe - Exit an unsafe region in the driver
 * @wctxt:        The write context that was handed to the write function
 *
 * Return:        True if this context still owns the console. False if
 *                ownership was handed over or taken.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context.
 */
bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        bool ret;

        ret = nbcon_context_exit_unsafe(ctxt);
        if (!ret)
                nbcon_write_context_set_buf(wctxt, NULL, 0);
        return ret;
}
EXPORT_SYMBOL_GPL(nbcon_exit_unsafe);

/**
 * nbcon_reacquire_nobuf - Reacquire a console after losing ownership
 *                                while printing
 * @wctxt:        The write context that was handed to the write callback
 *
 * Since ownership can be lost at any time due to handover or takeover, a
 * printing context _must_ be prepared to back out immediately and
 * carefully. However, there are scenarios where the printing context must
 * reacquire ownership in order to finalize or revert hardware changes.
 *
 * This function allows a printing context to reacquire ownership using the
 * same priority as its previous ownership.
 *
 * Note that after a successful reacquire the printing context will have no
 * output buffer because that has been lost. This function cannot be used to
 * resume printing.
 */
void nbcon_reacquire_nobuf(struct nbcon_write_context *wctxt)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);

        while (!nbcon_context_try_acquire(ctxt, true))
                cpu_relax();

        nbcon_write_context_set_buf(wctxt, NULL, 0);
}
EXPORT_SYMBOL_GPL(nbcon_reacquire_nobuf);

#ifdef CONFIG_PRINTK_EXECUTION_CTX
static void wctxt_load_execution_ctx(struct nbcon_write_context *wctxt,
                                     struct printk_message *pmsg)
{
        wctxt->cpu = pmsg->cpu;
        wctxt->pid = pmsg->pid;
        memcpy(wctxt->comm, pmsg->comm, sizeof(wctxt->comm));
        static_assert(sizeof(wctxt->comm) == sizeof(pmsg->comm));
}
#else
static void wctxt_load_execution_ctx(struct nbcon_write_context *wctxt,
                                     struct printk_message *pmsg) {}
#endif

/**
 * nbcon_emit_next_record - Emit a record in the acquired context
 * @wctxt:        The write context that will be handed to the write function
 * @use_atomic:        True if the write_atomic() callback is to be used
 *
 * Return:        True if this context still owns the console. False if
 *                ownership was handed over or taken.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context. If the caller
 * wants to do more it must reacquire the console first.
 *
 * When true is returned, @wctxt->ctxt.backlog indicates whether there are
 * still records pending in the ringbuffer,
 */
static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_atomic)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        struct console *con = ctxt->console;
        bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
        struct printk_message pmsg = {
                .pbufs = ctxt->pbufs,
        };
        unsigned long con_dropped;
        struct nbcon_state cur;
        unsigned long dropped;
        unsigned long ulseq;

        /*
         * This function should never be called for consoles that have not
         * implemented the necessary callback for writing: i.e. legacy
         * consoles and, when atomic, nbcon consoles with no write_atomic().
         * Handle it as if ownership was lost and try to continue.
         *
         * Note that for nbcon consoles the write_thread() callback is
         * mandatory and was already checked in nbcon_alloc().
         */
        if (WARN_ON_ONCE((use_atomic && !con->write_atomic) ||
                         !(console_srcu_read_flags(con) & CON_NBCON))) {
                nbcon_context_release(ctxt);
                return false;
        }

        /*
         * The printk buffers are filled within an unsafe section. This
         * prevents NBCON_PRIO_NORMAL and NBCON_PRIO_EMERGENCY from
         * clobbering each other.
         */

        if (!nbcon_context_enter_unsafe(ctxt))
                return false;

        ctxt->backlog = printk_get_next_message(&pmsg, ctxt->seq, is_extended, true);
        if (!ctxt->backlog)
                return nbcon_context_exit_unsafe(ctxt);

        /*
         * @con->dropped is not protected in case of an unsafe hostile
         * takeover. In that situation the update can be racy so
         * annotate it accordingly.
         */
        con_dropped = data_race(READ_ONCE(con->dropped));

        dropped = con_dropped + pmsg.dropped;
        if (dropped && !is_extended)
                console_prepend_dropped(&pmsg, dropped);

        /*
         * If the previous owner was assigned the same record, this context
         * has taken over ownership and is replaying the record. Prepend a
         * message to let the user know the record is replayed.
         */
        ulseq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_prev_seq));
        if (__ulseq_to_u64seq(prb, ulseq) == pmsg.seq) {
                console_prepend_replay(&pmsg);
        } else {
                /*
                 * Ensure this context is still the owner before trying to
                 * update @nbcon_prev_seq. Otherwise the value in @ulseq may
                 * not be from the previous owner and instead be some later
                 * value from the context that took over ownership.
                 */
                nbcon_state_read(con, &cur);
                if (!nbcon_context_can_proceed(ctxt, &cur))
                        return false;

                atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_prev_seq), &ulseq,
                                        __u64seq_to_ulseq(pmsg.seq));
        }

        if (!nbcon_context_exit_unsafe(ctxt))
                return false;

        /* For skipped records just update seq/dropped in @con. */
        if (pmsg.outbuf_len == 0)
                goto update_con;

        /* Initialize the write context for driver callbacks. */
        nbcon_write_context_set_buf(wctxt, &pmsg.pbufs->outbuf[0], pmsg.outbuf_len);

        wctxt_load_execution_ctx(wctxt, &pmsg);

        if (use_atomic)
                con->write_atomic(con, wctxt);
        else
                con->write_thread(con, wctxt);

        if (!wctxt->outbuf) {
                /*
                 * Ownership was lost and reacquired by the driver. Handle it
                 * as if ownership was lost.
                 */
                nbcon_context_release(ctxt);
                return false;
        }

        /*
         * Ownership may have been lost but _not_ reacquired by the driver.
         * This case is detected and handled when entering unsafe to update
         * dropped/seq values.
         */

        /*
         * Since any dropped message was successfully output, reset the
         * dropped count for the console.
         */
        dropped = 0;
update_con:
        /*
         * The dropped count and the sequence number are updated within an
         * unsafe section. This limits update races to the panic context and
         * allows the panic context to win.
         */

        if (!nbcon_context_enter_unsafe(ctxt))
                return false;

        if (dropped != con_dropped) {
                /* Counterpart to the READ_ONCE() above. */
                WRITE_ONCE(con->dropped, dropped);
        }

        nbcon_seq_try_update(ctxt, pmsg.seq + 1);

        return nbcon_context_exit_unsafe(ctxt);
}

/*
 * nbcon_emit_one - Print one record for an nbcon console using the
 *                        specified callback
 * @wctxt:        An initialized write context struct to use for this context
 * @use_atomic:        True if the write_atomic() callback is to be used
 *
 * Return:        True, when a record has been printed and there are still
 *                pending records. The caller might want to continue flushing.
 *
 *                False, when there is no pending record, or when the console
 *                context cannot be acquired, or the ownership has been lost.
 *                The caller should give up. Either the job is done, cannot be
 *                done, or will be handled by the owning context.
 *
 * This is an internal helper to handle the locking of the console before
 * calling nbcon_emit_next_record().
 */
static bool nbcon_emit_one(struct nbcon_write_context *wctxt, bool use_atomic)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        struct console *con = ctxt->console;
        unsigned long flags;
        bool ret = false;

        if (!use_atomic) {
                con->device_lock(con, &flags);

                /*
                 * Ensure this stays on the CPU to make handover and
                 * takeover possible.
                 */
                cant_migrate();
        }

        if (!nbcon_context_try_acquire(ctxt, false))
                goto out;

        /*
         * nbcon_emit_next_record() returns false when the console was
         * handed over or taken over. In both cases the context is no
         * longer valid.
         *
         * The higher priority printing context takes over responsibility
         * to print the pending records.
         */
        if (!nbcon_emit_next_record(wctxt, use_atomic))
                goto out;

        nbcon_context_release(ctxt);

        ret = ctxt->backlog;
out:
        if (!use_atomic)
                con->device_unlock(con, flags);
        return ret;
}

/**
 * nbcon_kthread_should_wakeup - Check whether a printer thread should wakeup
 * @con:        Console to operate on
 * @ctxt:        The nbcon context from nbcon_context_try_acquire()
 *
 * Return:        True if the thread should shutdown or if the console is
 *                allowed to print and a record is available. False otherwise.
 *
 * After the thread wakes up, it must first check if it should shutdown before
 * attempting any printing.
 */
static bool nbcon_kthread_should_wakeup(struct console *con, struct nbcon_context *ctxt)
{
        bool ret = false;
        short flags;
        int cookie;

        if (kthread_should_stop())
                return true;

        /*
         * Block the kthread when the system is in an emergency or panic mode.
         * It increases the chance that these contexts would be able to show
         * the messages directly. And it reduces the risk of interrupted writes
         * where the context with a higher priority takes over the nbcon console
         * ownership in the middle of a message.
         */
        if (unlikely(atomic_read(&nbcon_cpu_emergency_cnt)) ||
            unlikely(panic_in_progress()))
                return false;

        cookie = console_srcu_read_lock();

        flags = console_srcu_read_flags(con);
        if (console_is_usable(con, flags, false)) {
                /* Bring the sequence in @ctxt up to date */
                ctxt->seq = nbcon_seq_read(con);

                ret = prb_read_valid(prb, ctxt->seq, NULL);
        }

        console_srcu_read_unlock(cookie);
        return ret;
}

/**
 * nbcon_kthread_func - The printer thread function
 * @__console:        Console to operate on
 *
 * Return:        0
 */
static int nbcon_kthread_func(void *__console)
{
        struct console *con = __console;
        struct nbcon_write_context wctxt = {
                .ctxt.console        = con,
                .ctxt.prio        = NBCON_PRIO_NORMAL,
        };
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
        short con_flags;
        bool backlog;
        int cookie;

wait_for_event:
        /*
         * Guarantee this task is visible on the rcuwait before
         * checking the wake condition.
         *
         * The full memory barrier within set_current_state() of
         * ___rcuwait_wait_event() pairs with the full memory
         * barrier within rcuwait_has_sleeper().
         *
         * This pairs with rcuwait_has_sleeper:A and nbcon_kthread_wake:A.
         */
        rcuwait_wait_event(&con->rcuwait,
                           nbcon_kthread_should_wakeup(con, ctxt),
                           TASK_INTERRUPTIBLE); /* LMM(nbcon_kthread_func:A) */

        do {
                if (kthread_should_stop())
                        return 0;

                /*
                 * Block the kthread when the system is in an emergency or panic
                 * mode. See nbcon_kthread_should_wakeup() for more details.
                 */
                if (unlikely(atomic_read(&nbcon_cpu_emergency_cnt)) ||
                    unlikely(panic_in_progress()))
                        goto wait_for_event;

                backlog = false;

                /*
                 * Keep the srcu read lock around the entire operation so that
                 * synchronize_srcu() can guarantee that the kthread stopped
                 * or suspended printing.
                 */
                cookie = console_srcu_read_lock();

                con_flags = console_srcu_read_flags(con);

                if (console_is_usable(con, con_flags, false))
                        backlog = nbcon_emit_one(&wctxt, false);

                console_srcu_read_unlock(cookie);

                cond_resched();

        } while (backlog);

        goto wait_for_event;
}

/**
 * nbcon_irq_work - irq work to wake console printer thread
 * @irq_work:        The irq work to operate on
 */
static void nbcon_irq_work(struct irq_work *irq_work)
{
        struct console *con = container_of(irq_work, struct console, irq_work);

        nbcon_kthread_wake(con);
}

static inline bool rcuwait_has_sleeper(struct rcuwait *w)
{
        /*
         * Guarantee any new records can be seen by tasks preparing to wait
         * before this context checks if the rcuwait is empty.
         *
         * This full memory barrier pairs with the full memory barrier within
         * set_current_state() of ___rcuwait_wait_event(), which is called
         * after prepare_to_rcuwait() adds the waiter but before it has
         * checked the wait condition.
         *
         * This pairs with nbcon_kthread_func:A.
         */
        smp_mb(); /* LMM(rcuwait_has_sleeper:A) */
        return rcuwait_active(w);
}

/**
 * nbcon_kthreads_wake - Wake up printing threads using irq_work
 */
void nbcon_kthreads_wake(void)
{
        struct console *con;
        int cookie;

        if (!printk_kthreads_running)
                return;

        /*
         * It is not allowed to call this function when console irq_work
         * is blocked.
         */
        if (WARN_ON_ONCE(console_irqwork_blocked))
                return;

        cookie = console_srcu_read_lock();
        for_each_console_srcu(con) {
                if (!(console_srcu_read_flags(con) & CON_NBCON))
                        continue;

                /*
                 * Only schedule irq_work if the printing thread is
                 * actively waiting. If not waiting, the thread will
                 * notice by itself that it has work to do.
                 */
                if (rcuwait_has_sleeper(&con->rcuwait))
                        irq_work_queue(&con->irq_work);
        }
        console_srcu_read_unlock(cookie);
}

/*
 * nbcon_kthread_stop - Stop a console printer thread
 * @con:        Console to operate on
 */
void nbcon_kthread_stop(struct console *con)
{
        lockdep_assert_console_list_lock_held();

        if (!con->kthread)
                return;

        kthread_stop(con->kthread);
        con->kthread = NULL;
}

/**
 * nbcon_kthread_create - Create a console printer thread
 * @con:        Console to operate on
 *
 * Return:        True if the kthread was started or already exists.
 *                Otherwise false and @con must not be registered.
 *
 * This function is called when it will be expected that nbcon consoles are
 * flushed using the kthread. The messages printed with NBCON_PRIO_NORMAL
 * will be no longer flushed by the legacy loop. This is why failure must
 * be fatal for console registration.
 *
 * If @con was already registered and this function fails, @con must be
 * unregistered before the global state variable @printk_kthreads_running
 * can be set.
 */
bool nbcon_kthread_create(struct console *con)
{
        struct task_struct *kt;

        lockdep_assert_console_list_lock_held();

        if (con->kthread)
                return true;

        kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index);
        if (WARN_ON(IS_ERR(kt))) {
                con_printk(KERN_ERR, con, "failed to start printing thread\n");
                return false;
        }

        con->kthread = kt;

        /*
         * It is important that console printing threads are scheduled
         * shortly after a printk call and with generous runtime budgets.
         */
        sched_set_normal(con->kthread, -20);

        return true;
}

/* Track the nbcon emergency nesting per CPU. */
static DEFINE_PER_CPU(unsigned int, nbcon_pcpu_emergency_nesting);
static unsigned int early_nbcon_pcpu_emergency_nesting __initdata;

/**
 * nbcon_get_cpu_emergency_nesting - Get the per CPU emergency nesting pointer
 *
 * Context:        For reading, any context. For writing, any context which could
 *                not be migrated to another CPU.
 * Return:        Either a pointer to the per CPU emergency nesting counter of
 *                the current CPU or to the init data during early boot.
 *
 * The function is safe for reading per-CPU variables in any context because
 * preemption is disabled if the current CPU is in the emergency state. See
 * also nbcon_cpu_emergency_enter().
 */
static __ref unsigned int *nbcon_get_cpu_emergency_nesting(void)
{
        /*
         * The value of __printk_percpu_data_ready gets set in normal
         * context and before SMP initialization. As a result it could
         * never change while inside an nbcon emergency section.
         */
        if (!printk_percpu_data_ready())
                return &early_nbcon_pcpu_emergency_nesting;

        return raw_cpu_ptr(&nbcon_pcpu_emergency_nesting);
}

/**
 * nbcon_get_default_prio - The appropriate nbcon priority to use for nbcon
 *                                printing on the current CPU
 *
 * Context:        Any context.
 * Return:        The nbcon_prio to use for acquiring an nbcon console in this
 *                context for printing.
 *
 * The function is safe for reading per-CPU data in any context because
 * preemption is disabled if the current CPU is in the emergency or panic
 * state.
 */
enum nbcon_prio nbcon_get_default_prio(void)
{
        unsigned int *cpu_emergency_nesting;

        if (panic_on_this_cpu())
                return NBCON_PRIO_PANIC;

        cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
        if (*cpu_emergency_nesting)
                return NBCON_PRIO_EMERGENCY;

        return NBCON_PRIO_NORMAL;
}

/*
 * Track if it is allowed to perform unsafe hostile takeovers of console
 * ownership. When true, console drivers might perform unsafe actions while
 * printing. It is externally available via nbcon_allow_unsafe_takeover().
 */
static bool panic_nbcon_allow_unsafe_takeover;

/**
 * nbcon_allow_unsafe_takeover - Check if unsafe console takeovers are allowed
 *
 * Return:        True, when it is permitted to perform unsafe console printing
 *
 * This is also used by console_is_usable() to determine if it is allowed to
 * call write_atomic() callbacks flagged as unsafe (CON_NBCON_ATOMIC_UNSAFE).
 */
bool nbcon_allow_unsafe_takeover(void)
{
        return panic_on_this_cpu() && panic_nbcon_allow_unsafe_takeover;
}

/**
 * nbcon_legacy_emit_next_record - Print one record for an nbcon console
 *                                        in legacy contexts
 * @con:        The console to print on
 * @handover:        Will be set to true if a printk waiter has taken over the
 *                console_lock, in which case the caller is no longer holding
 *                both the console_lock and the SRCU read lock. Otherwise it
 *                is set to false.
 * @cookie:        The cookie from the SRCU read lock.
 * @use_atomic: Set true when called in an atomic or unknown context.
 *                It affects which nbcon callback will be used: write_atomic()
 *                or write_thread().
 *
 *                When false, the write_thread() callback is used and would be
 *                called in a preemptible context unless disabled by the
 *                device_lock. The legacy handover is not allowed in this mode.
 *
 * Context:        Any context except NMI.
 * Return:        True, when a record has been printed and there are still
 *                pending records. The caller might want to continue flushing.
 *
 *                False, when there is no pending record, or when the console
 *                context cannot be acquired, or the ownership has been lost.
 *                The caller should give up. Either the job is done, cannot be
 *                done, or will be handled by the owning context.
 *
 * This function is meant to be called by console_flush_all() to print records
 * on nbcon consoles from legacy context (printing via console unlocking).
 * Essentially it is the nbcon version of console_emit_next_record().
 */
bool nbcon_legacy_emit_next_record(struct console *con, bool *handover,
                                   int cookie, bool use_atomic)
{
        struct nbcon_write_context wctxt = { };
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
        unsigned long flags;
        bool progress;

        ctxt->console        = con;
        ctxt->prio        = nbcon_get_default_prio();

        if (use_atomic) {
                /*
                 * In an atomic or unknown context, use the same procedure as
                 * in console_emit_next_record(). It allows to handover.
                 */
                printk_safe_enter_irqsave(flags);
                console_lock_spinning_enable();
                stop_critical_timings();
        }

        progress = nbcon_emit_one(&wctxt, use_atomic);

        if (use_atomic) {
                start_critical_timings();
                *handover = console_lock_spinning_disable_and_check(cookie);
                printk_safe_exit_irqrestore(flags);
        } else {
                /* Non-atomic does not perform legacy spinning handovers. */
                *handover = false;
        }

        return progress;
}

/**
 * __nbcon_atomic_flush_pending_con - Flush specified nbcon console using its
 *                                        write_atomic() callback
 * @con:                        The nbcon console to flush
 * @stop_seq:                        Flush up until this record
 *
 * Return:        0 if @con was flushed up to @stop_seq Otherwise, error code on
 *                failure.
 *
 * Errors:
 *
 *        -EPERM:                Unable to acquire console ownership.
 *
 *        -EAGAIN:        Another context took over ownership while printing.
 *
 *        -ENOENT:        A record before @stop_seq is not available.
 *
 * If flushing up to @stop_seq was not successful, it only makes sense for the
 * caller to try again when -EAGAIN was returned. When -EPERM is returned,
 * this context is not allowed to acquire the console. When -ENOENT is
 * returned, it cannot be expected that the unfinalized record will become
 * available.
 */
static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq)
{
        struct nbcon_write_context wctxt = { };
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
        int err = 0;

        ctxt->console                        = con;
        ctxt->spinwait_max_us                = 2000;
        ctxt->prio                        = nbcon_get_default_prio();
        ctxt->allow_unsafe_takeover        = nbcon_allow_unsafe_takeover();

        while (nbcon_seq_read(con) < stop_seq) {
                /*
                 * Atomic flushing does not use console driver synchronization
                 * (i.e. it does not hold the port lock for uart consoles).
                 * Therefore IRQs must be disabled to avoid being interrupted
                 * and then calling into a driver that will deadlock trying
                 * to acquire console ownership.
                 */
                scoped_guard(irqsave) {
                        if (!nbcon_context_try_acquire(ctxt, false))
                                return -EPERM;

                        /*
                         * nbcon_emit_next_record() returns false when
                         * the console was handed over or taken over.
                         * In both cases the context is no longer valid.
                         */
                        if (!nbcon_emit_next_record(&wctxt, true))
                                return -EAGAIN;

                        nbcon_context_release(ctxt);
                }

                if (!ctxt->backlog) {
                        /* Are there reserved but not yet finalized records? */
                        if (nbcon_seq_read(con) < stop_seq)
                                err = -ENOENT;
                        break;
                }
        }

        return err;
}

/**
 * nbcon_atomic_flush_pending_con - Flush specified nbcon console using its
 *                                        write_atomic() callback
 * @con:                        The nbcon console to flush
 * @stop_seq:                        Flush up until this record
 *
 * This will stop flushing before @stop_seq if another context has ownership.
 * That context is then responsible for the flushing. Likewise, if new records
 * are added while this context was flushing and there is no other context
 * to handle the printing, this context must also flush those records.
 */
static void nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq)
{
        struct console_flush_type ft;
        int err;

again:
        err = __nbcon_atomic_flush_pending_con(con, stop_seq);

        /*
         * If there was a new owner (-EPERM, -EAGAIN), that context is
         * responsible for completing.
         *
         * Do not wait for records not yet finalized (-ENOENT) to avoid a
         * possible deadlock. They will either get flushed by the writer or
         * eventually skipped on panic CPU.
         */
        if (err)
                return;

        /*
         * If flushing was successful but more records are available, this
         * context must flush those remaining records if the printer thread
         * is not available do it.
         */
        printk_get_console_flush_type(&ft);
        if (!ft.nbcon_offload &&
            prb_read_valid(prb, nbcon_seq_read(con), NULL)) {
                stop_seq = prb_next_reserve_seq(prb);
                goto again;
        }
}

/**
 * __nbcon_atomic_flush_pending - Flush all nbcon consoles using their
 *                                        write_atomic() callback
 * @stop_seq:                        Flush up until this record
 */
static void __nbcon_atomic_flush_pending(u64 stop_seq)
{
        struct console *con;
        int cookie;

        cookie = console_srcu_read_lock();
        for_each_console_srcu(con) {
                short flags = console_srcu_read_flags(con);

                if (!(flags & CON_NBCON))
                        continue;

                if (!console_is_usable(con, flags, true))
                        continue;

                if (nbcon_seq_read(con) >= stop_seq)
                        continue;

                nbcon_atomic_flush_pending_con(con, stop_seq);
        }
        console_srcu_read_unlock(cookie);
}

/**
 * nbcon_atomic_flush_pending - Flush all nbcon consoles using their
 *                                write_atomic() callback
 *
 * Flush the backlog up through the currently newest record. Any new
 * records added while flushing will not be flushed if there is another
 * context available to handle the flushing. This is to avoid one CPU
 * printing unbounded because other CPUs continue to add records.
 */
void nbcon_atomic_flush_pending(void)
{
        __nbcon_atomic_flush_pending(prb_next_reserve_seq(prb));
}

/**
 * nbcon_atomic_flush_unsafe - Flush all nbcon consoles using their
 *        write_atomic() callback and allowing unsafe hostile takeovers
 *
 * Flush the backlog up through the currently newest record. Unsafe hostile
 * takeovers will be performed, if necessary.
 */
void nbcon_atomic_flush_unsafe(void)
{
        panic_nbcon_allow_unsafe_takeover = true;
        __nbcon_atomic_flush_pending(prb_next_reserve_seq(prb));
        panic_nbcon_allow_unsafe_takeover = false;
}

/**
 * nbcon_cpu_emergency_enter - Enter an emergency section where printk()
 *                                messages for that CPU are flushed directly
 *
 * Context:        Any context. Disables preemption.
 *
 * When within an emergency section, printk() calls will attempt to flush any
 * pending messages in the ringbuffer.
 */
void nbcon_cpu_emergency_enter(void)
{
        unsigned int *cpu_emergency_nesting;

        preempt_disable();

        atomic_inc(&nbcon_cpu_emergency_cnt);

        cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
        (*cpu_emergency_nesting)++;
}

/**
 * nbcon_cpu_emergency_exit - Exit an emergency section
 *
 * Context:        Within an emergency section. Enables preemption.
 */
void nbcon_cpu_emergency_exit(void)
{
        unsigned int *cpu_emergency_nesting;

        cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
        if (!WARN_ON_ONCE(*cpu_emergency_nesting == 0))
                (*cpu_emergency_nesting)--;

        /*
         * Wake up kthreads because there might be some pending messages
         * added by other CPUs with normal priority since the last flush
         * in the emergency context.
         */
        if (!WARN_ON_ONCE(atomic_read(&nbcon_cpu_emergency_cnt) == 0)) {
                if (atomic_dec_return(&nbcon_cpu_emergency_cnt) == 0) {
                        struct console_flush_type ft;

                        printk_get_console_flush_type(&ft);
                        if (ft.nbcon_offload)
                                nbcon_kthreads_wake();
                }
        }

        preempt_enable();
}

/**
 * nbcon_alloc - Allocate and init the nbcon console specific data
 * @con:        Console to initialize
 *
 * Return:        True if the console was fully allocated and initialized.
 *                Otherwise @con must not be registered.
 *
 * When allocation and init was successful, the console must be properly
 * freed using nbcon_free() once it is no longer needed.
 */
bool nbcon_alloc(struct console *con)
{
        struct nbcon_state state = { };

        /* Synchronize the kthread start. */
        lockdep_assert_console_list_lock_held();

        /* Check for mandatory nbcon callbacks. */
        if (WARN_ON(!con->write_thread ||
                    !con->device_lock ||
                    !con->device_unlock)) {
                return false;
        }

        rcuwait_init(&con->rcuwait);
        init_irq_work(&con->irq_work, nbcon_irq_work);
        atomic_long_set(&ACCESS_PRIVATE(con, nbcon_prev_seq), -1UL);
        nbcon_state_set(con, &state);

        /*
         * Initialize @nbcon_seq to the highest possible sequence number so
         * that practically speaking it will have nothing to print until a
         * desired initial sequence number has been set via nbcon_seq_force().
         */
        atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), ULSEQ_MAX(prb));

        if (con->flags & CON_BOOT) {
                /*
                 * Boot console printing is synchronized with legacy console
                 * printing, so boot consoles can share the same global printk
                 * buffers.
                 */
                con->pbufs = &printk_shared_pbufs;
        } else {
                con->pbufs = kmalloc_obj(*con->pbufs);
                if (!con->pbufs) {
                        con_printk(KERN_ERR, con, "failed to allocate printing buffer\n");
                        return false;
                }

                if (printk_kthreads_ready && !have_boot_console) {
                        if (!nbcon_kthread_create(con)) {
                                kfree(con->pbufs);
                                con->pbufs = NULL;
                                return false;
                        }

                        /* Might be the first kthread. */
                        printk_kthreads_running = true;
                }
        }

        return true;
}

/**
 * nbcon_free - Free and cleanup the nbcon console specific data
 * @con:        Console to free/cleanup nbcon data
 *
 * Important: @have_nbcon_console must be updated before calling
 *        this function. In particular, it can be set only when there
 *        is still another nbcon console registered.
 */
void nbcon_free(struct console *con)
{
        struct nbcon_state state = { };

        /* Synchronize the kthread stop. */
        lockdep_assert_console_list_lock_held();

        if (printk_kthreads_running) {
                nbcon_kthread_stop(con);

                /* Might be the last nbcon console.
                 *
                 * Do not rely on printk_kthreads_check_locked(). It is not
                 * called in some code paths, see nbcon_free() callers.
                 */
                if (!have_nbcon_console)
                        printk_kthreads_running = false;
        }

        nbcon_state_set(con, &state);

        /* Boot consoles share global printk buffers. */
        if (!(con->flags & CON_BOOT))
                kfree(con->pbufs);

        con->pbufs = NULL;
}

/**
 * nbcon_device_try_acquire - Try to acquire nbcon console and enter unsafe
 *                                section
 * @con:        The nbcon console to acquire
 *
 * Context:        Under the locking mechanism implemented in
 *                @con->device_lock() including disabling migration.
 * Return:        True if the console was acquired. False otherwise.
 *
 * Console drivers will usually use their own internal synchronization
 * mechanism to synchronize between console printing and non-printing
 * activities (such as setting baud rates). However, nbcon console drivers
 * supporting atomic consoles may also want to mark unsafe sections when
 * performing non-printing activities in order to synchronize against their
 * atomic_write() callback.
 *
 * This function acquires the nbcon console using priority NBCON_PRIO_NORMAL
 * and marks it unsafe for handover/takeover.
 */
bool nbcon_device_try_acquire(struct console *con)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt);

        cant_migrate();

        memset(ctxt, 0, sizeof(*ctxt));
        ctxt->console        = con;
        ctxt->prio        = NBCON_PRIO_NORMAL;

        if (!nbcon_context_try_acquire(ctxt, false))
                return false;

        if (!nbcon_context_enter_unsafe(ctxt))
                return false;

        return true;
}
EXPORT_SYMBOL_GPL(nbcon_device_try_acquire);

/**
 * nbcon_device_release - Exit unsafe section and release the nbcon console
 * @con:        The nbcon console acquired in nbcon_device_try_acquire()
 */
void nbcon_device_release(struct console *con)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt);
        struct console_flush_type ft;
        int cookie;

        if (!nbcon_context_exit_unsafe(ctxt))
                return;

        nbcon_context_release(ctxt);

        /*
         * This context must flush any new records added while the console
         * was locked if the printer thread is not available to do it. The
         * console_srcu_read_lock must be taken to ensure the console is
         * usable throughout flushing.
         */
        cookie = console_srcu_read_lock();
        printk_get_console_flush_type(&ft);
        if (console_is_usable(con, console_srcu_read_flags(con), true) &&
            !ft.nbcon_offload &&
            prb_read_valid(prb, nbcon_seq_read(con), NULL)) {
                /*
                 * If nbcon_atomic flushing is not available, fallback to
                 * using the legacy loop.
                 */
                if (ft.nbcon_atomic) {
                        __nbcon_atomic_flush_pending_con(con, prb_next_reserve_seq(prb));
                } else if (ft.legacy_direct) {
                        if (console_trylock())
                                console_unlock();
                } else if (ft.legacy_offload) {
                        defer_console_output();
                }
        }
        console_srcu_read_unlock(cookie);
}
EXPORT_SYMBOL_GPL(nbcon_device_release);

/**
 * nbcon_kdb_try_acquire - Try to acquire nbcon console and enter unsafe
 *                           section
 * @con:        The nbcon console to acquire
 * @wctxt:        The nbcon write context to be used on success
 *
 * Context:        Under console_srcu_read_lock() for emitting a single kdb message
 *                using the given con->write_atomic() callback. Can be called
 *                only when the console is usable at the moment.
 *
 * Return:        True if the console was acquired. False otherwise.
 *
 * kdb emits messages on consoles registered for printk() without
 * storing them into the ring buffer. It has to acquire the console
 * ownership so that it could call con->write_atomic() callback a safe way.
 *
 * This function acquires the nbcon console using priority NBCON_PRIO_EMERGENCY
 * and marks it unsafe for handover/takeover.
 */
bool nbcon_kdb_try_acquire(struct console *con,
                           struct nbcon_write_context *wctxt)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);

        memset(ctxt, 0, sizeof(*ctxt));
        ctxt->console = con;
        ctxt->prio    = NBCON_PRIO_EMERGENCY;

        if (!nbcon_context_try_acquire(ctxt, false))
                return false;

        if (!nbcon_context_enter_unsafe(ctxt))
                return false;

        return true;
}

/**
 * nbcon_kdb_release - Exit unsafe section and release the nbcon console
 *
 * @wctxt:        The nbcon write context initialized by a successful
 *                nbcon_kdb_try_acquire()
 */
void nbcon_kdb_release(struct nbcon_write_context *wctxt)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);

        if (!nbcon_context_exit_unsafe(ctxt))
                return;

        nbcon_context_release(ctxt);

        /*
         * Flush any new printk() messages added when the console was blocked.
         * Only the console used by the given write context was        blocked.
         * The console was locked only when the write_atomic() callback
         * was usable.
         */
        __nbcon_atomic_flush_pending_con(ctxt->console, prb_next_reserve_seq(prb));
}


























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 



































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 























































































































































    4 



    2 








    3 









    2 























    2 











































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * iSCSI transport class definitions
 *
 * Copyright (C) IBM Corporation, 2004
 * Copyright (C) Mike Christie, 2004 - 2005
 * Copyright (C) Dmitry Yusupov, 2004 - 2005
 * Copyright (C) Alex Aizman, 2004 - 2005
 */
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/bsg-lib.h>
#include <linux/idr.h>
#include <net/tcp.h>
#include <scsi/scsi.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_transport.h>
#include <scsi/scsi_transport_iscsi.h>
#include <scsi/iscsi_if.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_bsg_iscsi.h>

#define ISCSI_TRANSPORT_VERSION "2.0-870"

#define ISCSI_SEND_MAX_ALLOWED  10

#define CREATE_TRACE_POINTS
#include <trace/events/iscsi.h>

/*
 * Export tracepoint symbols to be used by other modules.
 */
EXPORT_TRACEPOINT_SYMBOL_GPL(iscsi_dbg_conn);
EXPORT_TRACEPOINT_SYMBOL_GPL(iscsi_dbg_eh);
EXPORT_TRACEPOINT_SYMBOL_GPL(iscsi_dbg_session);
EXPORT_TRACEPOINT_SYMBOL_GPL(iscsi_dbg_tcp);
EXPORT_TRACEPOINT_SYMBOL_GPL(iscsi_dbg_sw_tcp);

static int dbg_session;
module_param_named(debug_session, dbg_session, int,
                   S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(debug_session,
                 "Turn on debugging for sessions in scsi_transport_iscsi "
                 "module. Set to 1 to turn on, and zero to turn off. Default "
                 "is off.");

static int dbg_conn;
module_param_named(debug_conn, dbg_conn, int,
                   S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(debug_conn,
                 "Turn on debugging for connections in scsi_transport_iscsi "
                 "module. Set to 1 to turn on, and zero to turn off. Default "
                 "is off.");

#define ISCSI_DBG_TRANS_SESSION(_session, dbg_fmt, arg...)                \
        do {                                                                \
                if (dbg_session)                                        \
                        iscsi_cls_session_printk(KERN_INFO, _session,        \
                                                 "%s: " dbg_fmt,        \
                                                 __func__, ##arg);        \
                iscsi_dbg_trace(trace_iscsi_dbg_trans_session,                \
                                &(_session)->dev,                        \
                                "%s " dbg_fmt, __func__, ##arg);        \
        } while (0);

#define ISCSI_DBG_TRANS_CONN(_conn, dbg_fmt, arg...)                        \
        do {                                                                \
                if (dbg_conn)                                                \
                        iscsi_cls_conn_printk(KERN_INFO, _conn,                \
                                              "%s: " dbg_fmt,                \
                                              __func__, ##arg);                \
                iscsi_dbg_trace(trace_iscsi_dbg_trans_conn,                \
                                &(_conn)->dev,                                \
                                "%s " dbg_fmt, __func__, ##arg);        \
        } while (0);

struct iscsi_internal {
        struct scsi_transport_template t;
        struct iscsi_transport *iscsi_transport;
        struct list_head list;
        struct device dev;

        struct transport_container conn_cont;
        struct transport_container session_cont;
};

static DEFINE_IDR(iscsi_ep_idr);
static DEFINE_MUTEX(iscsi_ep_idr_mutex);

static atomic_t iscsi_session_nr; /* sysfs session id for next new session */

static struct workqueue_struct *iscsi_conn_cleanup_workq;

static DEFINE_IDA(iscsi_sess_ida);
/*
 * list of registered transports and lock that must
 * be held while accessing list. The iscsi_transport_lock must
 * be acquired after the rx_queue_mutex.
 */
static LIST_HEAD(iscsi_transports);
static DEFINE_SPINLOCK(iscsi_transport_lock);

#define to_iscsi_internal(tmpl) \
        container_of(tmpl, struct iscsi_internal, t)

#define dev_to_iscsi_internal(_dev) \
        container_of(_dev, struct iscsi_internal, dev)

static void iscsi_transport_release(struct device *dev)
{
        struct iscsi_internal *priv = dev_to_iscsi_internal(dev);
        kfree(priv);
}

/*
 * iscsi_transport_class represents the iscsi_transports that are
 * registered.
 */
static struct class iscsi_transport_class = {
        .name = "iscsi_transport",
        .dev_release = iscsi_transport_release,
};

static ssize_t
show_transport_handle(struct device *dev, struct device_attribute *attr,
                      char *buf)
{
        struct iscsi_internal *priv = dev_to_iscsi_internal(dev);

        if (!capable(CAP_SYS_ADMIN))
                return -EACCES;
        return sysfs_emit(buf, "%llu\n",
                  (unsigned long long)iscsi_handle(priv->iscsi_transport));
}
static DEVICE_ATTR(handle, S_IRUGO, show_transport_handle, NULL);

#define show_transport_attr(name, format)                                \
static ssize_t                                                                \
show_transport_##name(struct device *dev,                                 \
                      struct device_attribute *attr,char *buf)                \
{                                                                        \
        struct iscsi_internal *priv = dev_to_iscsi_internal(dev);        \
        return sysfs_emit(buf, format"\n", priv->iscsi_transport->name);\
}                                                                        \
static DEVICE_ATTR(name, S_IRUGO, show_transport_##name, NULL);

show_transport_attr(caps, "0x%x");

static struct attribute *iscsi_transport_attrs[] = {
        &dev_attr_handle.attr,
        &dev_attr_caps.attr,
        NULL,
};

static struct attribute_group iscsi_transport_group = {
        .attrs = iscsi_transport_attrs,
};

/*
 * iSCSI endpoint attrs
 */
#define iscsi_dev_to_endpoint(_dev) \
        container_of(_dev, struct iscsi_endpoint, dev)

#define ISCSI_ATTR(_prefix,_name,_mode,_show,_store)        \
struct device_attribute dev_attr_##_prefix##_##_name =        \
        __ATTR(_name,_mode,_show,_store)

static void iscsi_endpoint_release(struct device *dev)
{
        struct iscsi_endpoint *ep = iscsi_dev_to_endpoint(dev);

        mutex_lock(&iscsi_ep_idr_mutex);
        idr_remove(&iscsi_ep_idr, ep->id);
        mutex_unlock(&iscsi_ep_idr_mutex);

        kfree(ep);
}

static struct class iscsi_endpoint_class = {
        .name = "iscsi_endpoint",
        .dev_release = iscsi_endpoint_release,
};

static ssize_t
show_ep_handle(struct device *dev, struct device_attribute *attr, char *buf)
{
        struct iscsi_endpoint *ep = iscsi_dev_to_endpoint(dev);
        return sysfs_emit(buf, "%d\n", ep->id);
}
static ISCSI_ATTR(ep, handle, S_IRUGO, show_ep_handle, NULL);

static struct attribute *iscsi_endpoint_attrs[] = {
        &dev_attr_ep_handle.attr,
        NULL,
};

static struct attribute_group iscsi_endpoint_group = {
        .attrs = iscsi_endpoint_attrs,
};

struct iscsi_endpoint *
iscsi_create_endpoint(int dd_size)
{
        struct iscsi_endpoint *ep;
        int err, id;

        ep = kzalloc(sizeof(*ep) + dd_size, GFP_KERNEL);
        if (!ep)
                return NULL;

        mutex_lock(&iscsi_ep_idr_mutex);

        /*
         * First endpoint id should be 1 to comply with user space
         * applications (iscsid).
         */
        id = idr_alloc(&iscsi_ep_idr, ep, 1, -1, GFP_NOIO);
        if (id < 0) {
                mutex_unlock(&iscsi_ep_idr_mutex);
                printk(KERN_ERR "Could not allocate endpoint ID. Error %d.\n",
                       id);
                goto free_ep;
        }
        mutex_unlock(&iscsi_ep_idr_mutex);

        ep->id = id;
        ep->dev.class = &iscsi_endpoint_class;
        dev_set_name(&ep->dev, "ep-%d", id);
        err = device_register(&ep->dev);
        if (err)
                goto put_dev;

        err = sysfs_create_group(&ep->dev.kobj, &iscsi_endpoint_group);
        if (err)
                goto unregister_dev;

        if (dd_size)
                ep->dd_data = &ep[1];
        return ep;

unregister_dev:
        device_unregister(&ep->dev);
        return NULL;

put_dev:
        mutex_lock(&iscsi_ep_idr_mutex);
        idr_remove(&iscsi_ep_idr, id);
        mutex_unlock(&iscsi_ep_idr_mutex);
        put_device(&ep->dev);
        return NULL;
free_ep:
        kfree(ep);
        return NULL;
}
EXPORT_SYMBOL_GPL(iscsi_create_endpoint);

void iscsi_destroy_endpoint(struct iscsi_endpoint *ep)
{
        sysfs_remove_group(&ep->dev.kobj, &iscsi_endpoint_group);
        device_unregister(&ep->dev);
}
EXPORT_SYMBOL_GPL(iscsi_destroy_endpoint);

void iscsi_put_endpoint(struct iscsi_endpoint *ep)
{
        put_device(&ep->dev);
}
EXPORT_SYMBOL_GPL(iscsi_put_endpoint);

/**
 * iscsi_lookup_endpoint - get ep from handle
 * @handle: endpoint handle
 *
 * Caller must do a iscsi_put_endpoint.
 */
struct iscsi_endpoint *iscsi_lookup_endpoint(u64 handle)
{
        struct iscsi_endpoint *ep;

        mutex_lock(&iscsi_ep_idr_mutex);
        ep = idr_find(&iscsi_ep_idr, handle);
        if (!ep)
                goto unlock;

        get_device(&ep->dev);
unlock:
        mutex_unlock(&iscsi_ep_idr_mutex);
        return ep;
}
EXPORT_SYMBOL_GPL(iscsi_lookup_endpoint);

/*
 * Interface to display network param to sysfs
 */

static void iscsi_iface_release(struct device *dev)
{
        struct iscsi_iface *iface = iscsi_dev_to_iface(dev);
        struct device *parent = iface->dev.parent;

        kfree(iface);
        put_device(parent);
}


static struct class iscsi_iface_class = {
        .name = "iscsi_iface",
        .dev_release = iscsi_iface_release,
};

#define ISCSI_IFACE_ATTR(_prefix, _name, _mode, _show, _store)        \
struct device_attribute dev_attr_##_prefix##_##_name =                \
        __ATTR(_name, _mode, _show, _store)

/* iface attrs show */
#define iscsi_iface_attr_show(type, name, param_type, param)                \
static ssize_t                                                                \
show_##type##_##name(struct device *dev, struct device_attribute *attr,        \
                     char *buf)                                                \
{                                                                        \
        struct iscsi_iface *iface = iscsi_dev_to_iface(dev);                \
        struct iscsi_transport *t = iface->transport;                        \
        return t->get_iface_param(iface, param_type, param, buf);        \
}                                                                        \

#define iscsi_iface_net_attr(type, name, param)                                \
        iscsi_iface_attr_show(type, name, ISCSI_NET_PARAM, param)        \
static ISCSI_IFACE_ATTR(type, name, S_IRUGO, show_##type##_##name, NULL);

#define iscsi_iface_attr(type, name, param)                                \
        iscsi_iface_attr_show(type, name, ISCSI_IFACE_PARAM, param)        \
static ISCSI_IFACE_ATTR(type, name, S_IRUGO, show_##type##_##name, NULL);

/* generic read only ipv4 attribute */
iscsi_iface_net_attr(ipv4_iface, ipaddress, ISCSI_NET_PARAM_IPV4_ADDR);
iscsi_iface_net_attr(ipv4_iface, gateway, ISCSI_NET_PARAM_IPV4_GW);
iscsi_iface_net_attr(ipv4_iface, subnet, ISCSI_NET_PARAM_IPV4_SUBNET);
iscsi_iface_net_attr(ipv4_iface, bootproto, ISCSI_NET_PARAM_IPV4_BOOTPROTO);
iscsi_iface_net_attr(ipv4_iface, dhcp_dns_address_en,
                     ISCSI_NET_PARAM_IPV4_DHCP_DNS_ADDR_EN);
iscsi_iface_net_attr(ipv4_iface, dhcp_slp_da_info_en,
                     ISCSI_NET_PARAM_IPV4_DHCP_SLP_DA_EN);
iscsi_iface_net_attr(ipv4_iface, tos_en, ISCSI_NET_PARAM_IPV4_TOS_EN);
iscsi_iface_net_attr(ipv4_iface, tos, ISCSI_NET_PARAM_IPV4_TOS);
iscsi_iface_net_attr(ipv4_iface, grat_arp_en,
                     ISCSI_NET_PARAM_IPV4_GRAT_ARP_EN);
iscsi_iface_net_attr(ipv4_iface, dhcp_alt_client_id_en,
                     ISCSI_NET_PARAM_IPV4_DHCP_ALT_CLIENT_ID_EN);
iscsi_iface_net_attr(ipv4_iface, dhcp_alt_client_id,
                     ISCSI_NET_PARAM_IPV4_DHCP_ALT_CLIENT_ID);
iscsi_iface_net_attr(ipv4_iface, dhcp_req_vendor_id_en,
                     ISCSI_NET_PARAM_IPV4_DHCP_REQ_VENDOR_ID_EN);
iscsi_iface_net_attr(ipv4_iface, dhcp_use_vendor_id_en,
                     ISCSI_NET_PARAM_IPV4_DHCP_USE_VENDOR_ID_EN);
iscsi_iface_net_attr(ipv4_iface, dhcp_vendor_id,
                     ISCSI_NET_PARAM_IPV4_DHCP_VENDOR_ID);
iscsi_iface_net_attr(ipv4_iface, dhcp_learn_iqn_en,
                     ISCSI_NET_PARAM_IPV4_DHCP_LEARN_IQN_EN);
iscsi_iface_net_attr(ipv4_iface, fragment_disable,
                     ISCSI_NET_PARAM_IPV4_FRAGMENT_DISABLE);
iscsi_iface_net_attr(ipv4_iface, incoming_forwarding_en,
                     ISCSI_NET_PARAM_IPV4_IN_FORWARD_EN);
iscsi_iface_net_attr(ipv4_iface, ttl, ISCSI_NET_PARAM_IPV4_TTL);

/* generic read only ipv6 attribute */
iscsi_iface_net_attr(ipv6_iface, ipaddress, ISCSI_NET_PARAM_IPV6_ADDR);
iscsi_iface_net_attr(ipv6_iface, link_local_addr,
                     ISCSI_NET_PARAM_IPV6_LINKLOCAL);
iscsi_iface_net_attr(ipv6_iface, router_addr, ISCSI_NET_PARAM_IPV6_ROUTER);
iscsi_iface_net_attr(ipv6_iface, ipaddr_autocfg,
                     ISCSI_NET_PARAM_IPV6_ADDR_AUTOCFG);
iscsi_iface_net_attr(ipv6_iface, link_local_autocfg,
                     ISCSI_NET_PARAM_IPV6_LINKLOCAL_AUTOCFG);
iscsi_iface_net_attr(ipv6_iface, link_local_state,
                     ISCSI_NET_PARAM_IPV6_LINKLOCAL_STATE);
iscsi_iface_net_attr(ipv6_iface, router_state,
                     ISCSI_NET_PARAM_IPV6_ROUTER_STATE);
iscsi_iface_net_attr(ipv6_iface, grat_neighbor_adv_en,
                     ISCSI_NET_PARAM_IPV6_GRAT_NEIGHBOR_ADV_EN);
iscsi_iface_net_attr(ipv6_iface, mld_en, ISCSI_NET_PARAM_IPV6_MLD_EN);
iscsi_iface_net_attr(ipv6_iface, flow_label, ISCSI_NET_PARAM_IPV6_FLOW_LABEL);
iscsi_iface_net_attr(ipv6_iface, traffic_class,
                     ISCSI_NET_PARAM_IPV6_TRAFFIC_CLASS);
iscsi_iface_net_attr(ipv6_iface, hop_limit, ISCSI_NET_PARAM_IPV6_HOP_LIMIT);
iscsi_iface_net_attr(ipv6_iface, nd_reachable_tmo,
                     ISCSI_NET_PARAM_IPV6_ND_REACHABLE_TMO);
iscsi_iface_net_attr(ipv6_iface, nd_rexmit_time,
                     ISCSI_NET_PARAM_IPV6_ND_REXMIT_TIME);
iscsi_iface_net_attr(ipv6_iface, nd_stale_tmo,
                     ISCSI_NET_PARAM_IPV6_ND_STALE_TMO);
iscsi_iface_net_attr(ipv6_iface, dup_addr_detect_cnt,
                     ISCSI_NET_PARAM_IPV6_DUP_ADDR_DETECT_CNT);
iscsi_iface_net_attr(ipv6_iface, router_adv_link_mtu,
                     ISCSI_NET_PARAM_IPV6_RTR_ADV_LINK_MTU);

/* common read only iface attribute */
iscsi_iface_net_attr(iface, enabled, ISCSI_NET_PARAM_IFACE_ENABLE);
iscsi_iface_net_attr(iface, vlan_id, ISCSI_NET_PARAM_VLAN_ID);
iscsi_iface_net_attr(iface, vlan_priority, ISCSI_NET_PARAM_VLAN_PRIORITY);
iscsi_iface_net_attr(iface, vlan_enabled, ISCSI_NET_PARAM_VLAN_ENABLED);
iscsi_iface_net_attr(iface, mtu, ISCSI_NET_PARAM_MTU);
iscsi_iface_net_attr(iface, port, ISCSI_NET_PARAM_PORT);
iscsi_iface_net_attr(iface, ipaddress_state, ISCSI_NET_PARAM_IPADDR_STATE);
iscsi_iface_net_attr(iface, delayed_ack_en, ISCSI_NET_PARAM_DELAYED_ACK_EN);
iscsi_iface_net_attr(iface, tcp_nagle_disable,
                     ISCSI_NET_PARAM_TCP_NAGLE_DISABLE);
iscsi_iface_net_attr(iface, tcp_wsf_disable, ISCSI_NET_PARAM_TCP_WSF_DISABLE);
iscsi_iface_net_attr(iface, tcp_wsf, ISCSI_NET_PARAM_TCP_WSF);
iscsi_iface_net_attr(iface, tcp_timer_scale, ISCSI_NET_PARAM_TCP_TIMER_SCALE);
iscsi_iface_net_attr(iface, tcp_timestamp_en, ISCSI_NET_PARAM_TCP_TIMESTAMP_EN);
iscsi_iface_net_attr(iface, cache_id, ISCSI_NET_PARAM_CACHE_ID);
iscsi_iface_net_attr(iface, redirect_en, ISCSI_NET_PARAM_REDIRECT_EN);

/* common iscsi specific settings attributes */
iscsi_iface_attr(iface, def_taskmgmt_tmo, ISCSI_IFACE_PARAM_DEF_TASKMGMT_TMO);
iscsi_iface_attr(iface, header_digest, ISCSI_IFACE_PARAM_HDRDGST_EN);
iscsi_iface_attr(iface, data_digest, ISCSI_IFACE_PARAM_DATADGST_EN);
iscsi_iface_attr(iface, immediate_data, ISCSI_IFACE_PARAM_IMM_DATA_EN);
iscsi_iface_attr(iface, initial_r2t, ISCSI_IFACE_PARAM_INITIAL_R2T_EN);
iscsi_iface_attr(iface, data_seq_in_order,
                 ISCSI_IFACE_PARAM_DATASEQ_INORDER_EN);
iscsi_iface_attr(iface, data_pdu_in_order, ISCSI_IFACE_PARAM_PDU_INORDER_EN);
iscsi_iface_attr(iface, erl, ISCSI_IFACE_PARAM_ERL);
iscsi_iface_attr(iface, max_recv_dlength, ISCSI_IFACE_PARAM_MAX_RECV_DLENGTH);
iscsi_iface_attr(iface, first_burst_len, ISCSI_IFACE_PARAM_FIRST_BURST);
iscsi_iface_attr(iface, max_outstanding_r2t, ISCSI_IFACE_PARAM_MAX_R2T);
iscsi_iface_attr(iface, max_burst_len, ISCSI_IFACE_PARAM_MAX_BURST);
iscsi_iface_attr(iface, chap_auth, ISCSI_IFACE_PARAM_CHAP_AUTH_EN);
iscsi_iface_attr(iface, bidi_chap, ISCSI_IFACE_PARAM_BIDI_CHAP_EN);
iscsi_iface_attr(iface, discovery_auth_optional,
                 ISCSI_IFACE_PARAM_DISCOVERY_AUTH_OPTIONAL);
iscsi_iface_attr(iface, discovery_logout,
                 ISCSI_IFACE_PARAM_DISCOVERY_LOGOUT_EN);
iscsi_iface_attr(iface, strict_login_comp_en,
                 ISCSI_IFACE_PARAM_STRICT_LOGIN_COMP_EN);
iscsi_iface_attr(iface, initiator_name, ISCSI_IFACE_PARAM_INITIATOR_NAME);

static umode_t iscsi_iface_attr_is_visible(struct kobject *kobj,
                                          struct attribute *attr, int i)
{
        struct device *dev = container_of(kobj, struct device, kobj);
        struct iscsi_iface *iface = iscsi_dev_to_iface(dev);
        struct iscsi_transport *t = iface->transport;
        int param = -1;

        if (attr == &dev_attr_iface_def_taskmgmt_tmo.attr)
                param = ISCSI_IFACE_PARAM_DEF_TASKMGMT_TMO;
        else if (attr == &dev_attr_iface_header_digest.attr)
                param = ISCSI_IFACE_PARAM_HDRDGST_EN;
        else if (attr == &dev_attr_iface_data_digest.attr)
                param = ISCSI_IFACE_PARAM_DATADGST_EN;
        else if (attr == &dev_attr_iface_immediate_data.attr)
                param = ISCSI_IFACE_PARAM_IMM_DATA_EN;
        else if (attr == &dev_attr_iface_initial_r2t.attr)
                param = ISCSI_IFACE_PARAM_INITIAL_R2T_EN;
        else if (attr == &dev_attr_iface_data_seq_in_order.attr)
                param = ISCSI_IFACE_PARAM_DATASEQ_INORDER_EN;
        else if (attr == &dev_attr_iface_data_pdu_in_order.attr)
                param = ISCSI_IFACE_PARAM_PDU_INORDER_EN;
        else if (attr == &dev_attr_iface_erl.attr)
                param = ISCSI_IFACE_PARAM_ERL;
        else if (attr == &dev_attr_iface_max_recv_dlength.attr)
                param = ISCSI_IFACE_PARAM_MAX_RECV_DLENGTH;
        else if (attr == &dev_attr_iface_first_burst_len.attr)
                param = ISCSI_IFACE_PARAM_FIRST_BURST;
        else if (attr == &dev_attr_iface_max_outstanding_r2t.attr)
                param = ISCSI_IFACE_PARAM_MAX_R2T;
        else if (attr == &dev_attr_iface_max_burst_len.attr)
                param = ISCSI_IFACE_PARAM_MAX_BURST;
        else if (attr == &dev_attr_iface_chap_auth.attr)
                param = ISCSI_IFACE_PARAM_CHAP_AUTH_EN;
        else if (attr == &dev_attr_iface_bidi_chap.attr)
                param = ISCSI_IFACE_PARAM_BIDI_CHAP_EN;
        else if (attr == &dev_attr_iface_discovery_auth_optional.attr)
                param = ISCSI_IFACE_PARAM_DISCOVERY_AUTH_OPTIONAL;
        else if (attr == &dev_attr_iface_discovery_logout.attr)
                param = ISCSI_IFACE_PARAM_DISCOVERY_LOGOUT_EN;
        else if (attr == &dev_attr_iface_strict_login_comp_en.attr)
                param = ISCSI_IFACE_PARAM_STRICT_LOGIN_COMP_EN;
        else if (attr == &dev_attr_iface_initiator_name.attr)
                param = ISCSI_IFACE_PARAM_INITIATOR_NAME;

        if (param != -1)
                return t->attr_is_visible(ISCSI_IFACE_PARAM, param);

        if (attr == &dev_attr_iface_enabled.attr)
                param = ISCSI_NET_PARAM_IFACE_ENABLE;
        else if (attr == &dev_attr_iface_vlan_id.attr)
                param = ISCSI_NET_PARAM_VLAN_ID;
        else if (attr == &dev_attr_iface_vlan_priority.attr)
                param = ISCSI_NET_PARAM_VLAN_PRIORITY;
        else if (attr == &dev_attr_iface_vlan_enabled.attr)
                param = ISCSI_NET_PARAM_VLAN_ENABLED;
        else if (attr == &dev_attr_iface_mtu.attr)
                param = ISCSI_NET_PARAM_MTU;
        else if (attr == &dev_attr_iface_port.attr)
                param = ISCSI_NET_PARAM_PORT;
        else if (attr == &dev_attr_iface_ipaddress_state.attr)
                param = ISCSI_NET_PARAM_IPADDR_STATE;
        else if (attr == &dev_attr_iface_delayed_ack_en.attr)
                param = ISCSI_NET_PARAM_DELAYED_ACK_EN;
        else if (attr == &dev_attr_iface_tcp_nagle_disable.attr)
                param = ISCSI_NET_PARAM_TCP_NAGLE_DISABLE;
        else if (attr == &dev_attr_iface_tcp_wsf_disable.attr)
                param = ISCSI_NET_PARAM_TCP_WSF_DISABLE;
        else if (attr == &dev_attr_iface_tcp_wsf.attr)
                param = ISCSI_NET_PARAM_TCP_WSF;
        else if (attr == &dev_attr_iface_tcp_timer_scale.attr)
                param = ISCSI_NET_PARAM_TCP_TIMER_SCALE;
        else if (attr == &dev_attr_iface_tcp_timestamp_en.attr)
                param = ISCSI_NET_PARAM_TCP_TIMESTAMP_EN;
        else if (attr == &dev_attr_iface_cache_id.attr)
                param = ISCSI_NET_PARAM_CACHE_ID;
        else if (attr == &dev_attr_iface_redirect_en.attr)
                param = ISCSI_NET_PARAM_REDIRECT_EN;
        else if (iface->iface_type == ISCSI_IFACE_TYPE_IPV4) {
                if (attr == &dev_attr_ipv4_iface_ipaddress.attr)
                        param = ISCSI_NET_PARAM_IPV4_ADDR;
                else if (attr == &dev_attr_ipv4_iface_gateway.attr)
                        param = ISCSI_NET_PARAM_IPV4_GW;
                else if (attr == &dev_attr_ipv4_iface_subnet.attr)
                        param = ISCSI_NET_PARAM_IPV4_SUBNET;
                else if (attr == &dev_attr_ipv4_iface_bootproto.attr)
                        param = ISCSI_NET_PARAM_IPV4_BOOTPROTO;
                else if (attr ==
                         &dev_attr_ipv4_iface_dhcp_dns_address_en.attr)
                        param = ISCSI_NET_PARAM_IPV4_DHCP_DNS_ADDR_EN;
                else if (attr ==
                         &dev_attr_ipv4_iface_dhcp_slp_da_info_en.attr)
                        param = ISCSI_NET_PARAM_IPV4_DHCP_SLP_DA_EN;
                else if (attr == &dev_attr_ipv4_iface_tos_en.attr)
                        param = ISCSI_NET_PARAM_IPV4_TOS_EN;
                else if (attr == &dev_attr_ipv4_iface_tos.attr)
                        param = ISCSI_NET_PARAM_IPV4_TOS;
                else if (attr == &dev_attr_ipv4_iface_grat_arp_en.attr)
                        param = ISCSI_NET_PARAM_IPV4_GRAT_ARP_EN;
                else if (attr ==
                         &dev_attr_ipv4_iface_dhcp_alt_client_id_en.attr)
                        param = ISCSI_NET_PARAM_IPV4_DHCP_ALT_CLIENT_ID_EN;
                else if (attr == &dev_attr_ipv4_iface_dhcp_alt_client_id.attr)
                        param = ISCSI_NET_PARAM_IPV4_DHCP_ALT_CLIENT_ID;
                else if (attr ==
                         &dev_attr_ipv4_iface_dhcp_req_vendor_id_en.attr)
                        param = ISCSI_NET_PARAM_IPV4_DHCP_REQ_VENDOR_ID_EN;
                else if (attr ==
                         &dev_attr_ipv4_iface_dhcp_use_vendor_id_en.attr)
                        param = ISCSI_NET_PARAM_IPV4_DHCP_USE_VENDOR_ID_EN;
                else if (attr == &dev_attr_ipv4_iface_dhcp_vendor_id.attr)
                        param = ISCSI_NET_PARAM_IPV4_DHCP_VENDOR_ID;
                else if (attr ==
                         &dev_attr_ipv4_iface_dhcp_learn_iqn_en.attr)
                        param = ISCSI_NET_PARAM_IPV4_DHCP_LEARN_IQN_EN;
                else if (attr ==
                         &dev_attr_ipv4_iface_fragment_disable.attr)
                        param = ISCSI_NET_PARAM_IPV4_FRAGMENT_DISABLE;
                else if (attr ==
                         &dev_attr_ipv4_iface_incoming_forwarding_en.attr)
                        param = ISCSI_NET_PARAM_IPV4_IN_FORWARD_EN;
                else if (attr == &dev_attr_ipv4_iface_ttl.attr)
                        param = ISCSI_NET_PARAM_IPV4_TTL;
                else
                        return 0;
        } else if (iface->iface_type == ISCSI_IFACE_TYPE_IPV6) {
                if (attr == &dev_attr_ipv6_iface_ipaddress.attr)
                        param = ISCSI_NET_PARAM_IPV6_ADDR;
                else if (attr == &dev_attr_ipv6_iface_link_local_addr.attr)
                        param = ISCSI_NET_PARAM_IPV6_LINKLOCAL;
                else if (attr == &dev_attr_ipv6_iface_router_addr.attr)
                        param = ISCSI_NET_PARAM_IPV6_ROUTER;
                else if (attr == &dev_attr_ipv6_iface_ipaddr_autocfg.attr)
                        param = ISCSI_NET_PARAM_IPV6_ADDR_AUTOCFG;
                else if (attr == &dev_attr_ipv6_iface_link_local_autocfg.attr)
                        param = ISCSI_NET_PARAM_IPV6_LINKLOCAL_AUTOCFG;
                else if (attr == &dev_attr_ipv6_iface_link_local_state.attr)
                        param = ISCSI_NET_PARAM_IPV6_LINKLOCAL_STATE;
                else if (attr == &dev_attr_ipv6_iface_router_state.attr)
                        param = ISCSI_NET_PARAM_IPV6_ROUTER_STATE;
                else if (attr ==
                         &dev_attr_ipv6_iface_grat_neighbor_adv_en.attr)
                        param = ISCSI_NET_PARAM_IPV6_GRAT_NEIGHBOR_ADV_EN;
                else if (attr == &dev_attr_ipv6_iface_mld_en.attr)
                        param = ISCSI_NET_PARAM_IPV6_MLD_EN;
                else if (attr == &dev_attr_ipv6_iface_flow_label.attr)
                        param = ISCSI_NET_PARAM_IPV6_FLOW_LABEL;
                else if (attr == &dev_attr_ipv6_iface_traffic_class.attr)
                        param = ISCSI_NET_PARAM_IPV6_TRAFFIC_CLASS;
                else if (attr == &dev_attr_ipv6_iface_hop_limit.attr)
                        param = ISCSI_NET_PARAM_IPV6_HOP_LIMIT;
                else if (attr == &dev_attr_ipv6_iface_nd_reachable_tmo.attr)
                        param = ISCSI_NET_PARAM_IPV6_ND_REACHABLE_TMO;
                else if (attr == &dev_attr_ipv6_iface_nd_rexmit_time.attr)
                        param = ISCSI_NET_PARAM_IPV6_ND_REXMIT_TIME;
                else if (attr == &dev_attr_ipv6_iface_nd_stale_tmo.attr)
                        param = ISCSI_NET_PARAM_IPV6_ND_STALE_TMO;
                else if (attr == &dev_attr_ipv6_iface_dup_addr_detect_cnt.attr)
                        param = ISCSI_NET_PARAM_IPV6_DUP_ADDR_DETECT_CNT;
                else if (attr == &dev_attr_ipv6_iface_router_adv_link_mtu.attr)
                        param = ISCSI_NET_PARAM_IPV6_RTR_ADV_LINK_MTU;
                else
                        return 0;
        } else {
                WARN_ONCE(1, "Invalid iface attr");
                return 0;
        }

        return t->attr_is_visible(ISCSI_NET_PARAM, param);
}

static struct attribute *iscsi_iface_attrs[] = {
        &dev_attr_iface_enabled.attr,
        &dev_attr_iface_vlan_id.attr,
        &dev_attr_iface_vlan_priority.attr,
        &dev_attr_iface_vlan_enabled.attr,
        &dev_attr_ipv4_iface_ipaddress.attr,
        &dev_attr_ipv4_iface_gateway.attr,
        &dev_attr_ipv4_iface_subnet.attr,
        &dev_attr_ipv4_iface_bootproto.attr,
        &dev_attr_ipv6_iface_ipaddress.attr,
        &dev_attr_ipv6_iface_link_local_addr.attr,
        &dev_attr_ipv6_iface_router_addr.attr,
        &dev_attr_ipv6_iface_ipaddr_autocfg.attr,
        &dev_attr_ipv6_iface_link_local_autocfg.attr,
        &dev_attr_iface_mtu.attr,
        &dev_attr_iface_port.attr,
        &dev_attr_iface_ipaddress_state.attr,
        &dev_attr_iface_delayed_ack_en.attr,
        &dev_attr_iface_tcp_nagle_disable.attr,
        &dev_attr_iface_tcp_wsf_disable.attr,
        &dev_attr_iface_tcp_wsf.attr,
        &dev_attr_iface_tcp_timer_scale.attr,
        &dev_attr_iface_tcp_timestamp_en.attr,
        &dev_attr_iface_cache_id.attr,
        &dev_attr_iface_redirect_en.attr,
        &dev_attr_iface_def_taskmgmt_tmo.attr,
        &dev_attr_iface_header_digest.attr,
        &dev_attr_iface_data_digest.attr,
        &dev_attr_iface_immediate_data.attr,
        &dev_attr_iface_initial_r2t.attr,
        &dev_attr_iface_data_seq_in_order.attr,
        &dev_attr_iface_data_pdu_in_order.attr,
        &dev_attr_iface_erl.attr,
        &dev_attr_iface_max_recv_dlength.attr,
        &dev_attr_iface_first_burst_len.attr,
        &dev_attr_iface_max_outstanding_r2t.attr,
        &dev_attr_iface_max_burst_len.attr,
        &dev_attr_iface_chap_auth.attr,
        &dev_attr_iface_bidi_chap.attr,
        &dev_attr_iface_discovery_auth_optional.attr,
        &dev_attr_iface_discovery_logout.attr,
        &dev_attr_iface_strict_login_comp_en.attr,
        &dev_attr_iface_initiator_name.attr,
        &dev_attr_ipv4_iface_dhcp_dns_address_en.attr,
        &dev_attr_ipv4_iface_dhcp_slp_da_info_en.attr,
        &dev_attr_ipv4_iface_tos_en.attr,
        &dev_attr_ipv4_iface_tos.attr,
        &dev_attr_ipv4_iface_grat_arp_en.attr,
        &dev_attr_ipv4_iface_dhcp_alt_client_id_en.attr,
        &dev_attr_ipv4_iface_dhcp_alt_client_id.attr,
        &dev_attr_ipv4_iface_dhcp_req_vendor_id_en.attr,
        &dev_attr_ipv4_iface_dhcp_use_vendor_id_en.attr,
        &dev_attr_ipv4_iface_dhcp_vendor_id.attr,
        &dev_attr_ipv4_iface_dhcp_learn_iqn_en.attr,
        &dev_attr_ipv4_iface_fragment_disable.attr,
        &dev_attr_ipv4_iface_incoming_forwarding_en.attr,
        &dev_attr_ipv4_iface_ttl.attr,
        &dev_attr_ipv6_iface_link_local_state.attr,
        &dev_attr_ipv6_iface_router_state.attr,
        &dev_attr_ipv6_iface_grat_neighbor_adv_en.attr,
        &dev_attr_ipv6_iface_mld_en.attr,
        &dev_attr_ipv6_iface_flow_label.attr,
        &dev_attr_ipv6_iface_traffic_class.attr,
        &dev_attr_ipv6_iface_hop_limit.attr,
        &dev_attr_ipv6_iface_nd_reachable_tmo.attr,
        &dev_attr_ipv6_iface_nd_rexmit_time.attr,
        &dev_attr_ipv6_iface_nd_stale_tmo.attr,
        &dev_attr_ipv6_iface_dup_addr_detect_cnt.attr,
        &dev_attr_ipv6_iface_router_adv_link_mtu.attr,
        NULL,
};

static struct attribute_group iscsi_iface_group = {
        .attrs = iscsi_iface_attrs,
        .is_visible = iscsi_iface_attr_is_visible,
};

/* convert iscsi_ipaddress_state values to ascii string name */
static const struct {
        enum iscsi_ipaddress_state        value;
        char                                *name;
} iscsi_ipaddress_state_names[] = {
        {ISCSI_IPDDRESS_STATE_UNCONFIGURED,        "Unconfigured" },
        {ISCSI_IPDDRESS_STATE_ACQUIRING,        "Acquiring" },
        {ISCSI_IPDDRESS_STATE_TENTATIVE,        "Tentative" },
        {ISCSI_IPDDRESS_STATE_VALID,                "Valid" },
        {ISCSI_IPDDRESS_STATE_DISABLING,        "Disabling" },
        {ISCSI_IPDDRESS_STATE_INVALID,                "Invalid" },
        {ISCSI_IPDDRESS_STATE_DEPRECATED,        "Deprecated" },
};

char *iscsi_get_ipaddress_state_name(enum iscsi_ipaddress_state port_state)
{
        int i;
        char *state = NULL;

        for (i = 0; i < ARRAY_SIZE(iscsi_ipaddress_state_names); i++) {
                if (iscsi_ipaddress_state_names[i].value == port_state) {
                        state = iscsi_ipaddress_state_names[i].name;
                        break;
                }
        }
        return state;
}
EXPORT_SYMBOL_GPL(iscsi_get_ipaddress_state_name);

/* convert iscsi_router_state values to ascii string name */
static const struct {
        enum iscsi_router_state        value;
        char                        *name;
} iscsi_router_state_names[] = {
        {ISCSI_ROUTER_STATE_UNKNOWN,                "Unknown" },
        {ISCSI_ROUTER_STATE_ADVERTISED,                "Advertised" },
        {ISCSI_ROUTER_STATE_MANUAL,                "Manual" },
        {ISCSI_ROUTER_STATE_STALE,                "Stale" },
};

char *iscsi_get_router_state_name(enum iscsi_router_state router_state)
{
        int i;
        char *state = NULL;

        for (i = 0; i < ARRAY_SIZE(iscsi_router_state_names); i++) {
                if (iscsi_router_state_names[i].value == router_state) {
                        state = iscsi_router_state_names[i].name;
                        break;
                }
        }
        return state;
}
EXPORT_SYMBOL_GPL(iscsi_get_router_state_name);

struct iscsi_iface *
iscsi_create_iface(struct Scsi_Host *shost, struct iscsi_transport *transport,
                   uint32_t iface_type, uint32_t iface_num, int dd_size)
{
        struct iscsi_iface *iface;
        int err;

        iface = kzalloc(sizeof(*iface) + dd_size, GFP_KERNEL);
        if (!iface)
                return NULL;

        iface->transport = transport;
        iface->iface_type = iface_type;
        iface->iface_num = iface_num;
        iface->dev.release = iscsi_iface_release;
        iface->dev.class = &iscsi_iface_class;
        /* parent reference released in iscsi_iface_release */
        iface->dev.parent = get_device(&shost->shost_gendev);
        if (iface_type == ISCSI_IFACE_TYPE_IPV4)
                dev_set_name(&iface->dev, "ipv4-iface-%u-%u", shost->host_no,
                             iface_num);
        else
                dev_set_name(&iface->dev, "ipv6-iface-%u-%u", shost->host_no,
                             iface_num);

        err = device_register(&iface->dev);
        if (err)
                goto put_dev;

        err = sysfs_create_group(&iface->dev.kobj, &iscsi_iface_group);
        if (err)
                goto unreg_iface;

        if (dd_size)
                iface->dd_data = &iface[1];
        return iface;

unreg_iface:
        device_unregister(&iface->dev);
        return NULL;

put_dev:
        put_device(&iface->dev);
        return NULL;
}
EXPORT_SYMBOL_GPL(iscsi_create_iface);

void iscsi_destroy_iface(struct iscsi_iface *iface)
{
        sysfs_remove_group(&iface->dev.kobj, &iscsi_iface_group);
        device_unregister(&iface->dev);
}
EXPORT_SYMBOL_GPL(iscsi_destroy_iface);

/*
 * Interface to display flash node params to sysfs
 */

#define ISCSI_FLASHNODE_ATTR(_prefix, _name, _mode, _show, _store)        \
struct device_attribute dev_attr_##_prefix##_##_name =                        \
        __ATTR(_name, _mode, _show, _store)

/* flash node session attrs show */
#define iscsi_flashnode_sess_attr_show(type, name, param)                \
static ssize_t                                                                \
show_##type##_##name(struct device *dev, struct device_attribute *attr,        \
                     char *buf)                                                \
{                                                                        \
        struct iscsi_bus_flash_session *fnode_sess =                        \
                                        iscsi_dev_to_flash_session(dev);\
        struct iscsi_transport *t = fnode_sess->transport;                \
        return t->get_flashnode_param(fnode_sess, param, buf);                \
}                                                                        \


#define iscsi_flashnode_sess_attr(type, name, param)                        \
        iscsi_flashnode_sess_attr_show(type, name, param)                \
static ISCSI_FLASHNODE_ATTR(type, name, S_IRUGO,                        \
                            show_##type##_##name, NULL);

/* Flash node session attributes */

iscsi_flashnode_sess_attr(fnode, auto_snd_tgt_disable,
                          ISCSI_FLASHNODE_AUTO_SND_TGT_DISABLE);
iscsi_flashnode_sess_attr(fnode, discovery_session,
                          ISCSI_FLASHNODE_DISCOVERY_SESS);
iscsi_flashnode_sess_attr(fnode, portal_type, ISCSI_FLASHNODE_PORTAL_TYPE);
iscsi_flashnode_sess_attr(fnode, entry_enable, ISCSI_FLASHNODE_ENTRY_EN);
iscsi_flashnode_sess_attr(fnode, immediate_data, ISCSI_FLASHNODE_IMM_DATA_EN);
iscsi_flashnode_sess_attr(fnode, initial_r2t, ISCSI_FLASHNODE_INITIAL_R2T_EN);
iscsi_flashnode_sess_attr(fnode, data_seq_in_order,
                          ISCSI_FLASHNODE_DATASEQ_INORDER);
iscsi_flashnode_sess_attr(fnode, data_pdu_in_order,
                          ISCSI_FLASHNODE_PDU_INORDER);
iscsi_flashnode_sess_attr(fnode, chap_auth, ISCSI_FLASHNODE_CHAP_AUTH_EN);
iscsi_flashnode_sess_attr(fnode, discovery_logout,
                          ISCSI_FLASHNODE_DISCOVERY_LOGOUT_EN);
iscsi_flashnode_sess_attr(fnode, bidi_chap, ISCSI_FLASHNODE_BIDI_CHAP_EN);
iscsi_flashnode_sess_attr(fnode, discovery_auth_optional,
                          ISCSI_FLASHNODE_DISCOVERY_AUTH_OPTIONAL);
iscsi_flashnode_sess_attr(fnode, erl, ISCSI_FLASHNODE_ERL);
iscsi_flashnode_sess_attr(fnode, first_burst_len, ISCSI_FLASHNODE_FIRST_BURST);
iscsi_flashnode_sess_attr(fnode, def_time2wait, ISCSI_FLASHNODE_DEF_TIME2WAIT);
iscsi_flashnode_sess_attr(fnode, def_time2retain,
                          ISCSI_FLASHNODE_DEF_TIME2RETAIN);
iscsi_flashnode_sess_attr(fnode, max_outstanding_r2t, ISCSI_FLASHNODE_MAX_R2T);
iscsi_flashnode_sess_attr(fnode, isid, ISCSI_FLASHNODE_ISID);
iscsi_flashnode_sess_attr(fnode, tsid, ISCSI_FLASHNODE_TSID);
iscsi_flashnode_sess_attr(fnode, max_burst_len, ISCSI_FLASHNODE_MAX_BURST);
iscsi_flashnode_sess_attr(fnode, def_taskmgmt_tmo,
                          ISCSI_FLASHNODE_DEF_TASKMGMT_TMO);
iscsi_flashnode_sess_attr(fnode, targetalias, ISCSI_FLASHNODE_ALIAS);
iscsi_flashnode_sess_attr(fnode, targetname, ISCSI_FLASHNODE_NAME);
iscsi_flashnode_sess_attr(fnode, tpgt, ISCSI_FLASHNODE_TPGT);
iscsi_flashnode_sess_attr(fnode, discovery_parent_idx,
                          ISCSI_FLASHNODE_DISCOVERY_PARENT_IDX);
iscsi_flashnode_sess_attr(fnode, discovery_parent_type,
                          ISCSI_FLASHNODE_DISCOVERY_PARENT_TYPE);
iscsi_flashnode_sess_attr(fnode, chap_in_idx, ISCSI_FLASHNODE_CHAP_IN_IDX);
iscsi_flashnode_sess_attr(fnode, chap_out_idx, ISCSI_FLASHNODE_CHAP_OUT_IDX);
iscsi_flashnode_sess_attr(fnode, username, ISCSI_FLASHNODE_USERNAME);
iscsi_flashnode_sess_attr(fnode, username_in, ISCSI_FLASHNODE_USERNAME_IN);
iscsi_flashnode_sess_attr(fnode, password, ISCSI_FLASHNODE_PASSWORD);
iscsi_flashnode_sess_attr(fnode, password_in, ISCSI_FLASHNODE_PASSWORD_IN);
iscsi_flashnode_sess_attr(fnode, is_boot_target, ISCSI_FLASHNODE_IS_BOOT_TGT);

static struct attribute *iscsi_flashnode_sess_attrs[] = {
        &dev_attr_fnode_auto_snd_tgt_disable.attr,
        &dev_attr_fnode_discovery_session.attr,
        &dev_attr_fnode_portal_type.attr,
        &dev_attr_fnode_entry_enable.attr,
        &dev_attr_fnode_immediate_data.attr,
        &dev_attr_fnode_initial_r2t.attr,
        &dev_attr_fnode_data_seq_in_order.attr,
        &dev_attr_fnode_data_pdu_in_order.attr,
        &dev_attr_fnode_chap_auth.attr,
        &dev_attr_fnode_discovery_logout.attr,
        &dev_attr_fnode_bidi_chap.attr,
        &dev_attr_fnode_discovery_auth_optional.attr,
        &dev_attr_fnode_erl.attr,
        &dev_attr_fnode_first_burst_len.attr,
        &dev_attr_fnode_def_time2wait.attr,
        &dev_attr_fnode_def_time2retain.attr,
        &dev_attr_fnode_max_outstanding_r2t.attr,
        &dev_attr_fnode_isid.attr,
        &dev_attr_fnode_tsid.attr,
        &dev_attr_fnode_max_burst_len.attr,
        &dev_attr_fnode_def_taskmgmt_tmo.attr,
        &dev_attr_fnode_targetalias.attr,
        &dev_attr_fnode_targetname.attr,
        &dev_attr_fnode_tpgt.attr,
        &dev_attr_fnode_discovery_parent_idx.attr,
        &dev_attr_fnode_discovery_parent_type.attr,
        &dev_attr_fnode_chap_in_idx.attr,
        &dev_attr_fnode_chap_out_idx.attr,
        &dev_attr_fnode_username.attr,
        &dev_attr_fnode_username_in.attr,
        &dev_attr_fnode_password.attr,
        &dev_attr_fnode_password_in.attr,
        &dev_attr_fnode_is_boot_target.attr,
        NULL,
};

static umode_t iscsi_flashnode_sess_attr_is_visible(struct kobject *kobj,
                                                    struct attribute *attr,
                                                    int i)
{
        struct device *dev = container_of(kobj, struct device, kobj);
        struct iscsi_bus_flash_session *fnode_sess =
                                                iscsi_dev_to_flash_session(dev);
        struct iscsi_transport *t = fnode_sess->transport;
        int param;

        if (attr == &dev_attr_fnode_auto_snd_tgt_disable.attr) {
                param = ISCSI_FLASHNODE_AUTO_SND_TGT_DISABLE;
        } else if (attr == &dev_attr_fnode_discovery_session.attr) {
                param = ISCSI_FLASHNODE_DISCOVERY_SESS;
        } else if (attr == &dev_attr_fnode_portal_type.attr) {
                param = ISCSI_FLASHNODE_PORTAL_TYPE;
        } else if (attr == &dev_attr_fnode_entry_enable.attr) {
                param = ISCSI_FLASHNODE_ENTRY_EN;
        } else if (attr == &dev_attr_fnode_immediate_data.attr) {
                param = ISCSI_FLASHNODE_IMM_DATA_EN;
        } else if (attr == &dev_attr_fnode_initial_r2t.attr) {
                param = ISCSI_FLASHNODE_INITIAL_R2T_EN;
        } else if (attr == &dev_attr_fnode_data_seq_in_order.attr) {
                param = ISCSI_FLASHNODE_DATASEQ_INORDER;
        } else if (attr == &dev_attr_fnode_data_pdu_in_order.attr) {
                param = ISCSI_FLASHNODE_PDU_INORDER;
        } else if (attr == &dev_attr_fnode_chap_auth.attr) {
                param = ISCSI_FLASHNODE_CHAP_AUTH_EN;
        } else if (attr == &dev_attr_fnode_discovery_logout.attr) {
                param = ISCSI_FLASHNODE_DISCOVERY_LOGOUT_EN;
        } else if (attr == &dev_attr_fnode_bidi_chap.attr) {
                param = ISCSI_FLASHNODE_BIDI_CHAP_EN;
        } else if (attr == &dev_attr_fnode_discovery_auth_optional.attr) {
                param = ISCSI_FLASHNODE_DISCOVERY_AUTH_OPTIONAL;
        } else if (attr == &dev_attr_fnode_erl.attr) {
                param = ISCSI_FLASHNODE_ERL;
        } else if (attr == &dev_attr_fnode_first_burst_len.attr) {
                param = ISCSI_FLASHNODE_FIRST_BURST;
        } else if (attr == &dev_attr_fnode_def_time2wait.attr) {
                param = ISCSI_FLASHNODE_DEF_TIME2WAIT;
        } else if (attr == &dev_attr_fnode_def_time2retain.attr) {
                param = ISCSI_FLASHNODE_DEF_TIME2RETAIN;
        } else if (attr == &dev_attr_fnode_max_outstanding_r2t.attr) {
                param = ISCSI_FLASHNODE_MAX_R2T;
        } else if (attr == &dev_attr_fnode_isid.attr) {
                param = ISCSI_FLASHNODE_ISID;
        } else if (attr == &dev_attr_fnode_tsid.attr) {
                param = ISCSI_FLASHNODE_TSID;
        } else if (attr == &dev_attr_fnode_max_burst_len.attr) {
                param = ISCSI_FLASHNODE_MAX_BURST;
        } else if (attr == &dev_attr_fnode_def_taskmgmt_tmo.attr) {
                param = ISCSI_FLASHNODE_DEF_TASKMGMT_TMO;
        } else if (attr == &dev_attr_fnode_targetalias.attr) {
                param = ISCSI_FLASHNODE_ALIAS;
        } else if (attr == &dev_attr_fnode_targetname.attr) {
                param = ISCSI_FLASHNODE_NAME;
        } else if (attr == &dev_attr_fnode_tpgt.attr) {
                param = ISCSI_FLASHNODE_TPGT;
        } else if (attr == &dev_attr_fnode_discovery_parent_idx.attr) {
                param = ISCSI_FLASHNODE_DISCOVERY_PARENT_IDX;
        } else if (attr == &dev_attr_fnode_discovery_parent_type.attr) {
                param = ISCSI_FLASHNODE_DISCOVERY_PARENT_TYPE;
        } else if (attr == &dev_attr_fnode_chap_in_idx.attr) {
                param = ISCSI_FLASHNODE_CHAP_IN_IDX;
        } else if (attr == &dev_attr_fnode_chap_out_idx.attr) {
                param = ISCSI_FLASHNODE_CHAP_OUT_IDX;
        } else if (attr == &dev_attr_fnode_username.attr) {
                param = ISCSI_FLASHNODE_USERNAME;
        } else if (attr == &dev_attr_fnode_username_in.attr) {
                param = ISCSI_FLASHNODE_USERNAME_IN;
        } else if (attr == &dev_attr_fnode_password.attr) {
                param = ISCSI_FLASHNODE_PASSWORD;
        } else if (attr == &dev_attr_fnode_password_in.attr) {
                param = ISCSI_FLASHNODE_PASSWORD_IN;
        } else if (attr == &dev_attr_fnode_is_boot_target.attr) {
                param = ISCSI_FLASHNODE_IS_BOOT_TGT;
        } else {
                WARN_ONCE(1, "Invalid flashnode session attr");
                return 0;
        }

        return t->attr_is_visible(ISCSI_FLASHNODE_PARAM, param);
}

static struct attribute_group iscsi_flashnode_sess_attr_group = {
        .attrs = iscsi_flashnode_sess_attrs,
        .is_visible = iscsi_flashnode_sess_attr_is_visible,
};

static const struct attribute_group *iscsi_flashnode_sess_attr_groups[] = {
        &iscsi_flashnode_sess_attr_group,
        NULL,
};

static void iscsi_flashnode_sess_release(struct device *dev)
{
        struct iscsi_bus_flash_session *fnode_sess =
                                                iscsi_dev_to_flash_session(dev);

        kfree(fnode_sess->targetname);
        kfree(fnode_sess->targetalias);
        kfree(fnode_sess->portal_type);
        kfree(fnode_sess);
}

static const struct device_type iscsi_flashnode_sess_dev_type = {
        .name = "iscsi_flashnode_sess_dev_type",
        .groups = iscsi_flashnode_sess_attr_groups,
        .release = iscsi_flashnode_sess_release,
};

/* flash node connection attrs show */
#define iscsi_flashnode_conn_attr_show(type, name, param)                \
static ssize_t                                                                \
show_##type##_##name(struct device *dev, struct device_attribute *attr,        \
                     char *buf)                                                \
{                                                                        \
        struct iscsi_bus_flash_conn *fnode_conn = iscsi_dev_to_flash_conn(dev);\
        struct iscsi_bus_flash_session *fnode_sess =                        \
                                iscsi_flash_conn_to_flash_session(fnode_conn);\
        struct iscsi_transport *t = fnode_conn->transport;                \
        return t->get_flashnode_param(fnode_sess, param, buf);                \
}                                                                        \


#define iscsi_flashnode_conn_attr(type, name, param)                        \
        iscsi_flashnode_conn_attr_show(type, name, param)                \
static ISCSI_FLASHNODE_ATTR(type, name, S_IRUGO,                        \
                            show_##type##_##name, NULL);

/* Flash node connection attributes */

iscsi_flashnode_conn_attr(fnode, is_fw_assigned_ipv6,
                          ISCSI_FLASHNODE_IS_FW_ASSIGNED_IPV6);
iscsi_flashnode_conn_attr(fnode, header_digest, ISCSI_FLASHNODE_HDR_DGST_EN);
iscsi_flashnode_conn_attr(fnode, data_digest, ISCSI_FLASHNODE_DATA_DGST_EN);
iscsi_flashnode_conn_attr(fnode, snack_req, ISCSI_FLASHNODE_SNACK_REQ_EN);
iscsi_flashnode_conn_attr(fnode, tcp_timestamp_stat,
                          ISCSI_FLASHNODE_TCP_TIMESTAMP_STAT);
iscsi_flashnode_conn_attr(fnode, tcp_nagle_disable,
                          ISCSI_FLASHNODE_TCP_NAGLE_DISABLE);
iscsi_flashnode_conn_attr(fnode, tcp_wsf_disable,
                          ISCSI_FLASHNODE_TCP_WSF_DISABLE);
iscsi_flashnode_conn_attr(fnode, tcp_timer_scale,
                          ISCSI_FLASHNODE_TCP_TIMER_SCALE);
iscsi_flashnode_conn_attr(fnode, tcp_timestamp_enable,
                          ISCSI_FLASHNODE_TCP_TIMESTAMP_EN);
iscsi_flashnode_conn_attr(fnode, fragment_disable,
                          ISCSI_FLASHNODE_IP_FRAG_DISABLE);
iscsi_flashnode_conn_attr(fnode, keepalive_tmo, ISCSI_FLASHNODE_KEEPALIVE_TMO);
iscsi_flashnode_conn_attr(fnode, port, ISCSI_FLASHNODE_PORT);
iscsi_flashnode_conn_attr(fnode, ipaddress, ISCSI_FLASHNODE_IPADDR);
iscsi_flashnode_conn_attr(fnode, max_recv_dlength,
                          ISCSI_FLASHNODE_MAX_RECV_DLENGTH);
iscsi_flashnode_conn_attr(fnode, max_xmit_dlength,
                          ISCSI_FLASHNODE_MAX_XMIT_DLENGTH);
iscsi_flashnode_conn_attr(fnode, local_port, ISCSI_FLASHNODE_LOCAL_PORT);
iscsi_flashnode_conn_attr(fnode, ipv4_tos, ISCSI_FLASHNODE_IPV4_TOS);
iscsi_flashnode_conn_attr(fnode, ipv6_traffic_class, ISCSI_FLASHNODE_IPV6_TC);
iscsi_flashnode_conn_attr(fnode, ipv6_flow_label,
                          ISCSI_FLASHNODE_IPV6_FLOW_LABEL);
iscsi_flashnode_conn_attr(fnode, redirect_ipaddr,
                          ISCSI_FLASHNODE_REDIRECT_IPADDR);
iscsi_flashnode_conn_attr(fnode, max_segment_size,
                          ISCSI_FLASHNODE_MAX_SEGMENT_SIZE);
iscsi_flashnode_conn_attr(fnode, link_local_ipv6,
                          ISCSI_FLASHNODE_LINK_LOCAL_IPV6);
iscsi_flashnode_conn_attr(fnode, tcp_xmit_wsf, ISCSI_FLASHNODE_TCP_XMIT_WSF);
iscsi_flashnode_conn_attr(fnode, tcp_recv_wsf, ISCSI_FLASHNODE_TCP_RECV_WSF);
iscsi_flashnode_conn_attr(fnode, statsn, ISCSI_FLASHNODE_STATSN);
iscsi_flashnode_conn_attr(fnode, exp_statsn, ISCSI_FLASHNODE_EXP_STATSN);

static struct attribute *iscsi_flashnode_conn_attrs[] = {
        &dev_attr_fnode_is_fw_assigned_ipv6.attr,
        &dev_attr_fnode_header_digest.attr,
        &dev_attr_fnode_data_digest.attr,
        &dev_attr_fnode_snack_req.attr,
        &dev_attr_fnode_tcp_timestamp_stat.attr,
        &dev_attr_fnode_tcp_nagle_disable.attr,
        &dev_attr_fnode_tcp_wsf_disable.attr,
        &dev_attr_fnode_tcp_timer_scale.attr,
        &dev_attr_fnode_tcp_timestamp_enable.attr,
        &dev_attr_fnode_fragment_disable.attr,
        &dev_attr_fnode_max_recv_dlength.attr,
        &dev_attr_fnode_max_xmit_dlength.attr,
        &dev_attr_fnode_keepalive_tmo.attr,
        &dev_attr_fnode_port.attr,
        &dev_attr_fnode_ipaddress.attr,
        &dev_attr_fnode_redirect_ipaddr.attr,
        &dev_attr_fnode_max_segment_size.attr,
        &dev_attr_fnode_local_port.attr,
        &dev_attr_fnode_ipv4_tos.attr,
        &dev_attr_fnode_ipv6_traffic_class.attr,
        &dev_attr_fnode_ipv6_flow_label.attr,
        &dev_attr_fnode_link_local_ipv6.attr,
        &dev_attr_fnode_tcp_xmit_wsf.attr,
        &dev_attr_fnode_tcp_recv_wsf.attr,
        &dev_attr_fnode_statsn.attr,
        &dev_attr_fnode_exp_statsn.attr,
        NULL,
};

static umode_t iscsi_flashnode_conn_attr_is_visible(struct kobject *kobj,
                                                    struct attribute *attr,
                                                    int i)
{
        struct device *dev = container_of(kobj, struct device, kobj);
        struct iscsi_bus_flash_conn *fnode_conn = iscsi_dev_to_flash_conn(dev);
        struct iscsi_transport *t = fnode_conn->transport;
        int param;

        if (attr == &dev_attr_fnode_is_fw_assigned_ipv6.attr) {
                param = ISCSI_FLASHNODE_IS_FW_ASSIGNED_IPV6;
        } else if (attr == &dev_attr_fnode_header_digest.attr) {
                param = ISCSI_FLASHNODE_HDR_DGST_EN;
        } else if (attr == &dev_attr_fnode_data_digest.attr) {
                param = ISCSI_FLASHNODE_DATA_DGST_EN;
        } else if (attr == &dev_attr_fnode_snack_req.attr) {
                param = ISCSI_FLASHNODE_SNACK_REQ_EN;
        } else if (attr == &dev_attr_fnode_tcp_timestamp_stat.attr) {
                param = ISCSI_FLASHNODE_TCP_TIMESTAMP_STAT;
        } else if (attr == &dev_attr_fnode_tcp_nagle_disable.attr) {
                param = ISCSI_FLASHNODE_TCP_NAGLE_DISABLE;
        } else if (attr == &dev_attr_fnode_tcp_wsf_disable.attr) {
                param = ISCSI_FLASHNODE_TCP_WSF_DISABLE;
        } else if (attr == &dev_attr_fnode_tcp_timer_scale.attr) {
                param = ISCSI_FLASHNODE_TCP_TIMER_SCALE;
        } else if (attr == &dev_attr_fnode_tcp_timestamp_enable.attr) {
                param = ISCSI_FLASHNODE_TCP_TIMESTAMP_EN;
        } else if (attr == &dev_attr_fnode_fragment_disable.attr) {
                param = ISCSI_FLASHNODE_IP_FRAG_DISABLE;
        } else if (attr == &dev_attr_fnode_max_recv_dlength.attr) {
                param = ISCSI_FLASHNODE_MAX_RECV_DLENGTH;
        } else if (attr == &dev_attr_fnode_max_xmit_dlength.attr) {
                param = ISCSI_FLASHNODE_MAX_XMIT_DLENGTH;
        } else if (attr == &dev_attr_fnode_keepalive_tmo.attr) {
                param = ISCSI_FLASHNODE_KEEPALIVE_TMO;
        } else if (attr == &dev_attr_fnode_port.attr) {
                param = ISCSI_FLASHNODE_PORT;
        } else if (attr == &dev_attr_fnode_ipaddress.attr) {
                param = ISCSI_FLASHNODE_IPADDR;
        } else if (attr == &dev_attr_fnode_redirect_ipaddr.attr) {
                param = ISCSI_FLASHNODE_REDIRECT_IPADDR;
        } else if (attr == &dev_attr_fnode_max_segment_size.attr) {
                param = ISCSI_FLASHNODE_MAX_SEGMENT_SIZE;
        } else if (attr == &dev_attr_fnode_local_port.attr) {
                param = ISCSI_FLASHNODE_LOCAL_PORT;
        } else if (attr == &dev_attr_fnode_ipv4_tos.attr) {
                param = ISCSI_FLASHNODE_IPV4_TOS;
        } else if (attr == &dev_attr_fnode_ipv6_traffic_class.attr) {
                param = ISCSI_FLASHNODE_IPV6_TC;
        } else if (attr == &dev_attr_fnode_ipv6_flow_label.attr) {
                param = ISCSI_FLASHNODE_IPV6_FLOW_LABEL;
        } else if (attr == &dev_attr_fnode_link_local_ipv6.attr) {
                param = ISCSI_FLASHNODE_LINK_LOCAL_IPV6;
        } else if (attr == &dev_attr_fnode_tcp_xmit_wsf.attr) {
                param = ISCSI_FLASHNODE_TCP_XMIT_WSF;
        } else if (attr == &dev_attr_fnode_tcp_recv_wsf.attr) {
                param = ISCSI_FLASHNODE_TCP_RECV_WSF;
        } else if (attr == &dev_attr_fnode_statsn.attr) {
                param = ISCSI_FLASHNODE_STATSN;
        } else if (attr == &dev_attr_fnode_exp_statsn.attr) {
                param = ISCSI_FLASHNODE_EXP_STATSN;
        } else {
                WARN_ONCE(1, "Invalid flashnode connection attr");
                return 0;
        }

        return t->attr_is_visible(ISCSI_FLASHNODE_PARAM, param);
}

static struct attribute_group iscsi_flashnode_conn_attr_group = {
        .attrs = iscsi_flashnode_conn_attrs,
        .is_visible = iscsi_flashnode_conn_attr_is_visible,
};

static const struct attribute_group *iscsi_flashnode_conn_attr_groups[] = {
        &iscsi_flashnode_conn_attr_group,
        NULL,
};

static void iscsi_flashnode_conn_release(struct device *dev)
{
        struct iscsi_bus_flash_conn *fnode_conn = iscsi_dev_to_flash_conn(dev);

        kfree(fnode_conn->ipaddress);
        kfree(fnode_conn->redirect_ipaddr);
        kfree(fnode_conn->link_local_ipv6_addr);
        kfree(fnode_conn);
}

static const struct device_type iscsi_flashnode_conn_dev_type = {
        .name = "iscsi_flashnode_conn_dev_type",
        .groups = iscsi_flashnode_conn_attr_groups,
        .release = iscsi_flashnode_conn_release,
};

static const struct bus_type iscsi_flashnode_bus;

int iscsi_flashnode_bus_match(struct device *dev,
                              const struct device_driver *drv)
{
        if (dev->bus == &iscsi_flashnode_bus)
                return 1;
        return 0;
}
EXPORT_SYMBOL_GPL(iscsi_flashnode_bus_match);

static const struct bus_type iscsi_flashnode_bus = {
        .name = "iscsi_flashnode",
        .match = &iscsi_flashnode_bus_match,
};

/**
 * iscsi_create_flashnode_sess - Add flashnode session entry in sysfs
 * @shost: pointer to host data
 * @index: index of flashnode to add in sysfs
 * @transport: pointer to transport data
 * @dd_size: total size to allocate
 *
 * Adds a sysfs entry for the flashnode session attributes
 *
 * Returns:
 *  pointer to allocated flashnode sess on success
 *  %NULL on failure
 */
struct iscsi_bus_flash_session *
iscsi_create_flashnode_sess(struct Scsi_Host *shost, int index,
                            struct iscsi_transport *transport,
                            int dd_size)
{
        struct iscsi_bus_flash_session *fnode_sess;
        int err;

        fnode_sess = kzalloc(sizeof(*fnode_sess) + dd_size, GFP_KERNEL);
        if (!fnode_sess)
                return NULL;

        fnode_sess->transport = transport;
        fnode_sess->target_id = index;
        fnode_sess->dev.type = &iscsi_flashnode_sess_dev_type;
        fnode_sess->dev.bus = &iscsi_flashnode_bus;
        fnode_sess->dev.parent = &shost->shost_gendev;
        dev_set_name(&fnode_sess->dev, "flashnode_sess-%u:%u",
                     shost->host_no, index);

        err = device_register(&fnode_sess->dev);
        if (err)
                goto put_dev;

        if (dd_size)
                fnode_sess->dd_data = &fnode_sess[1];

        return fnode_sess;

put_dev:
        put_device(&fnode_sess->dev);
        return NULL;
}
EXPORT_SYMBOL_GPL(iscsi_create_flashnode_sess);

/**
 * iscsi_create_flashnode_conn - Add flashnode conn entry in sysfs
 * @shost: pointer to host data
 * @fnode_sess: pointer to the parent flashnode session entry
 * @transport: pointer to transport data
 * @dd_size: total size to allocate
 *
 * Adds a sysfs entry for the flashnode connection attributes
 *
 * Returns:
 *  pointer to allocated flashnode conn on success
 *  %NULL on failure
 */
struct iscsi_bus_flash_conn *
iscsi_create_flashnode_conn(struct Scsi_Host *shost,
                            struct iscsi_bus_flash_session *fnode_sess,
                            struct iscsi_transport *transport,
                            int dd_size)
{
        struct iscsi_bus_flash_conn *fnode_conn;
        int err;

        fnode_conn = kzalloc(sizeof(*fnode_conn) + dd_size, GFP_KERNEL);
        if (!fnode_conn)
                return NULL;

        fnode_conn->transport = transport;
        fnode_conn->dev.type = &iscsi_flashnode_conn_dev_type;
        fnode_conn->dev.bus = &iscsi_flashnode_bus;
        fnode_conn->dev.parent = &fnode_sess->dev;
        dev_set_name(&fnode_conn->dev, "flashnode_conn-%u:%u:0",
                     shost->host_no, fnode_sess->target_id);

        err = device_register(&fnode_conn->dev);
        if (err)
                goto put_dev;

        if (dd_size)
                fnode_conn->dd_data = &fnode_conn[1];

        return fnode_conn;

put_dev:
        put_device(&fnode_conn->dev);
        return NULL;
}
EXPORT_SYMBOL_GPL(iscsi_create_flashnode_conn);

/**
 * iscsi_is_flashnode_conn_dev - verify passed device is to be flashnode conn
 * @dev: device to verify
 * @data: pointer to data containing value to use for verification
 *
 * Verifies if the passed device is flashnode conn device
 *
 * Returns:
 *  1 on success
 *  0 on failure
 */
static int iscsi_is_flashnode_conn_dev(struct device *dev, const void *data)
{
        return dev->bus == &iscsi_flashnode_bus;
}

static int iscsi_destroy_flashnode_conn(struct iscsi_bus_flash_conn *fnode_conn)
{
        device_unregister(&fnode_conn->dev);
        return 0;
}

static int flashnode_match_index(struct device *dev, const void *data)
{
        struct iscsi_bus_flash_session *fnode_sess = NULL;
        int ret = 0;

        if (!iscsi_flashnode_bus_match(dev, NULL))
                goto exit_match_index;

        fnode_sess = iscsi_dev_to_flash_session(dev);
        ret = (fnode_sess->target_id == *((const int *)data)) ? 1 : 0;

exit_match_index:
        return ret;
}

/**
 * iscsi_get_flashnode_by_index -finds flashnode session entry by index
 * @shost: pointer to host data
 * @idx: index to match
 *
 * Finds the flashnode session object for the passed index
 *
 * Returns:
 *  pointer to found flashnode session object on success
 *  %NULL on failure
 */
static struct iscsi_bus_flash_session *
iscsi_get_flashnode_by_index(struct Scsi_Host *shost, uint32_t idx)
{
        struct iscsi_bus_flash_session *fnode_sess = NULL;
        struct device *dev;

        dev = device_find_child(&shost->shost_gendev, &idx,
                                flashnode_match_index);
        if (dev)
                fnode_sess = iscsi_dev_to_flash_session(dev);

        return fnode_sess;
}

/**
 * iscsi_find_flashnode_sess - finds flashnode session entry
 * @shost: pointer to host data
 * @data: pointer to data containing value to use for comparison
 * @fn: function pointer that does actual comparison
 *
 * Finds the flashnode session object comparing the data passed using logic
 * defined in passed function pointer
 *
 * Returns:
 *  pointer to found flashnode session device object on success
 *  %NULL on failure
 */
struct device *
iscsi_find_flashnode_sess(struct Scsi_Host *shost, const void *data,
                          device_match_t fn)
{
        return device_find_child(&shost->shost_gendev, data, fn);
}
EXPORT_SYMBOL_GPL(iscsi_find_flashnode_sess);

/**
 * iscsi_find_flashnode_conn - finds flashnode connection entry
 * @fnode_sess: pointer to parent flashnode session entry
 *
 * Finds the flashnode connection object comparing the data passed using logic
 * defined in passed function pointer
 *
 * Returns:
 *  pointer to found flashnode connection device object on success
 *  %NULL on failure
 */
struct device *
iscsi_find_flashnode_conn(struct iscsi_bus_flash_session *fnode_sess)
{
        return device_find_child(&fnode_sess->dev, NULL,
                                 iscsi_is_flashnode_conn_dev);
}
EXPORT_SYMBOL_GPL(iscsi_find_flashnode_conn);

static int iscsi_iter_destroy_flashnode_conn_fn(struct device *dev, void *data)
{
        if (!iscsi_is_flashnode_conn_dev(dev, NULL))
                return 0;

        return iscsi_destroy_flashnode_conn(iscsi_dev_to_flash_conn(dev));
}

/**
 * iscsi_destroy_flashnode_sess - destroy flashnode session entry
 * @fnode_sess: pointer to flashnode session entry to be destroyed
 *
 * Deletes the flashnode session entry and all children flashnode connection
 * entries from sysfs
 */
void iscsi_destroy_flashnode_sess(struct iscsi_bus_flash_session *fnode_sess)
{
        int err;

        err = device_for_each_child(&fnode_sess->dev, NULL,
                                    iscsi_iter_destroy_flashnode_conn_fn);
        if (err)
                pr_err("Could not delete all connections for %s. Error %d.\n",
                       fnode_sess->dev.kobj.name, err);

        device_unregister(&fnode_sess->dev);
}
EXPORT_SYMBOL_GPL(iscsi_destroy_flashnode_sess);

static int iscsi_iter_destroy_flashnode_fn(struct device *dev, void *data)
{
        if (!iscsi_flashnode_bus_match(dev, NULL))
                return 0;

        iscsi_destroy_flashnode_sess(iscsi_dev_to_flash_session(dev));
        return 0;
}

/**
 * iscsi_destroy_all_flashnode - destroy all flashnode session entries
 * @shost: pointer to host data
 *
 * Destroys all the flashnode session entries and all corresponding children
 * flashnode connection entries from sysfs
 */
void iscsi_destroy_all_flashnode(struct Scsi_Host *shost)
{
        device_for_each_child(&shost->shost_gendev, NULL,
                              iscsi_iter_destroy_flashnode_fn);
}
EXPORT_SYMBOL_GPL(iscsi_destroy_all_flashnode);

/*
 * BSG support
 */
/**
 * iscsi_bsg_host_dispatch - Dispatch command to LLD.
 * @job: bsg job to be processed
 */
static int iscsi_bsg_host_dispatch(struct bsg_job *job)
{
        struct Scsi_Host *shost = iscsi_job_to_shost(job);
        struct iscsi_bsg_request *req = job->request;
        struct iscsi_bsg_reply *reply = job->reply;
        struct iscsi_internal *i = to_iscsi_internal(shost->transportt);
        int cmdlen = sizeof(uint32_t);        /* start with length of msgcode */
        int ret;

        /* check if we have the msgcode value at least */
        if (job->request_len < sizeof(uint32_t)) {
                ret = -ENOMSG;
                goto fail_host_msg;
        }

        /* Validate the host command */
        switch (req->msgcode) {
        case ISCSI_BSG_HST_VENDOR:
                cmdlen += sizeof(struct iscsi_bsg_host_vendor);
                if ((shost->hostt->vendor_id == 0L) ||
                    (req->rqst_data.h_vendor.vendor_id !=
                        shost->hostt->vendor_id)) {
                        ret = -ESRCH;
                        goto fail_host_msg;
                }
                break;
        default:
                ret = -EBADR;
                goto fail_host_msg;
        }

        /* check if we really have all the request data needed */
        if (job->request_len < cmdlen) {
                ret = -ENOMSG;
                goto fail_host_msg;
        }

        ret = i->iscsi_transport->bsg_request(job);
        if (!ret)
                return 0;

fail_host_msg:
        /* return the errno failure code as the only status */
        BUG_ON(job->reply_len < sizeof(uint32_t));
        reply->reply_payload_rcv_len = 0;
        reply->result = ret;
        job->reply_len = sizeof(uint32_t);
        bsg_job_done(job, ret, 0);
        return 0;
}

/**
 * iscsi_bsg_host_add - Create and add the bsg hooks to receive requests
 * @shost: shost for iscsi_host
 * @ihost: iscsi_cls_host adding the structures to
 */
static int
iscsi_bsg_host_add(struct Scsi_Host *shost, struct iscsi_cls_host *ihost)
{
        struct device *dev = &shost->shost_gendev;
        struct iscsi_internal *i = to_iscsi_internal(shost->transportt);
        struct queue_limits lim;
        struct request_queue *q;
        char bsg_name[20];

        if (!i->iscsi_transport->bsg_request)
                return -ENOTSUPP;

        snprintf(bsg_name, sizeof(bsg_name), "iscsi_host%d", shost->host_no);
        scsi_init_limits(shost, &lim);
        q = bsg_setup_queue(dev, bsg_name, &lim, iscsi_bsg_host_dispatch, NULL,
                        0);
        if (IS_ERR(q)) {
                shost_printk(KERN_ERR, shost, "bsg interface failed to "
                             "initialize - no request queue\n");
                return PTR_ERR(q);
        }

        ihost->bsg_q = q;
        return 0;
}

static int iscsi_setup_host(struct transport_container *tc, struct device *dev,
                            struct device *cdev)
{
        struct Scsi_Host *shost = dev_to_shost(dev);
        struct iscsi_cls_host *ihost = shost->shost_data;

        memset(ihost, 0, sizeof(*ihost));
        mutex_init(&ihost->mutex);

        iscsi_bsg_host_add(shost, ihost);
        /* ignore any bsg add error - we just can't do sgio */

        return 0;
}

static int iscsi_remove_host(struct transport_container *tc,
                             struct device *dev, struct device *cdev)
{
        struct Scsi_Host *shost = dev_to_shost(dev);
        struct iscsi_cls_host *ihost = shost->shost_data;

        bsg_remove_queue(ihost->bsg_q);
        return 0;
}

static DECLARE_TRANSPORT_CLASS(iscsi_host_class,
                               "iscsi_host",
                               iscsi_setup_host,
                               iscsi_remove_host,
                               NULL);

static DECLARE_TRANSPORT_CLASS(iscsi_session_class,
                               "iscsi_session",
                               NULL,
                               NULL,
                               NULL);

static DECLARE_TRANSPORT_CLASS(iscsi_connection_class,
                               "iscsi_connection",
                               NULL,
                               NULL,
                               NULL);

static struct sock *nls;
static DEFINE_MUTEX(rx_queue_mutex);

static LIST_HEAD(sesslist);
static DEFINE_SPINLOCK(sesslock);
static LIST_HEAD(connlist);
static DEFINE_SPINLOCK(connlock);

static uint32_t iscsi_conn_get_sid(struct iscsi_cls_conn *conn)
{
        struct iscsi_cls_session *sess = iscsi_dev_to_session(conn->dev.parent);
        return sess->sid;
}

/*
 * Returns the matching session to a given sid
 */
static struct iscsi_cls_session *iscsi_session_lookup(uint32_t sid)
{
        unsigned long flags;
        struct iscsi_cls_session *sess;

        spin_lock_irqsave(&sesslock, flags);
        list_for_each_entry(sess, &sesslist, sess_list) {
                if (sess->sid == sid) {
                        spin_unlock_irqrestore(&sesslock, flags);
                        return sess;
                }
        }
        spin_unlock_irqrestore(&sesslock, flags);
        return NULL;
}

/*
 * Returns the matching connection to a given sid / cid tuple
 */
static struct iscsi_cls_conn *iscsi_conn_lookup(uint32_t sid, uint32_t cid)
{
        unsigned long flags;
        struct iscsi_cls_conn *conn;

        spin_lock_irqsave(&connlock, flags);
        list_for_each_entry(conn, &connlist, conn_list) {
                if ((conn->cid == cid) && (iscsi_conn_get_sid(conn) == sid)) {
                        spin_unlock_irqrestore(&connlock, flags);
                        return conn;
                }
        }
        spin_unlock_irqrestore(&connlock, flags);
        return NULL;
}

/*
 * The following functions can be used by LLDs that allocate
 * their own scsi_hosts or by software iscsi LLDs
 */
static struct {
        int value;
        char *name;
} iscsi_session_state_names[] = {
        { ISCSI_SESSION_LOGGED_IN,        "LOGGED_IN" },
        { ISCSI_SESSION_FAILED,                "FAILED" },
        { ISCSI_SESSION_FREE,                "FREE" },
};

static const char *iscsi_session_state_name(int state)
{
        int i;
        char *name = NULL;

        for (i = 0; i < ARRAY_SIZE(iscsi_session_state_names); i++) {
                if (iscsi_session_state_names[i].value == state) {
                        name = iscsi_session_state_names[i].name;
                        break;
                }
        }
        return name;
}

static char *iscsi_session_target_state_name[] = {
        [ISCSI_SESSION_TARGET_UNBOUND]   = "UNBOUND",
        [ISCSI_SESSION_TARGET_ALLOCATED] = "ALLOCATED",
        [ISCSI_SESSION_TARGET_SCANNED]   = "SCANNED",
        [ISCSI_SESSION_TARGET_UNBINDING] = "UNBINDING",
};

int iscsi_session_chkready(struct iscsi_cls_session *session)
{
        int err;

        switch (session->state) {
        case ISCSI_SESSION_LOGGED_IN:
                err = 0;
                break;
        case ISCSI_SESSION_FAILED:
                err = DID_IMM_RETRY << 16;
                break;
        case ISCSI_SESSION_FREE:
                err = DID_TRANSPORT_FAILFAST << 16;
                break;
        default:
                err = DID_NO_CONNECT << 16;
                break;
        }
        return err;
}
EXPORT_SYMBOL_GPL(iscsi_session_chkready);

int iscsi_is_session_online(struct iscsi_cls_session *session)
{
        unsigned long flags;
        int ret = 0;

        spin_lock_irqsave(&session->lock, flags);
        if (session->state == ISCSI_SESSION_LOGGED_IN)
                ret = 1;
        spin_unlock_irqrestore(&session->lock, flags);
        return ret;
}
EXPORT_SYMBOL_GPL(iscsi_is_session_online);

static void iscsi_session_release(struct device *dev)
{
        struct iscsi_cls_session *session = iscsi_dev_to_session(dev);
        struct Scsi_Host *shost;

        shost = iscsi_session_to_shost(session);
        scsi_host_put(shost);
        ISCSI_DBG_TRANS_SESSION(session, "Completing session release\n");
        kfree(session);
}

int iscsi_is_session_dev(const struct device *dev)
{
        return dev->release == iscsi_session_release;
}
EXPORT_SYMBOL_GPL(iscsi_is_session_dev);

static int iscsi_iter_session_fn(struct device *dev, void *data)
{
        void (* fn) (struct iscsi_cls_session *) = data;

        if (!iscsi_is_session_dev(dev))
                return 0;
        fn(iscsi_dev_to_session(dev));
        return 0;
}

void iscsi_host_for_each_session(struct Scsi_Host *shost,
                                 void (*fn)(struct iscsi_cls_session *))
{
        device_for_each_child(&shost->shost_gendev, fn,
                              iscsi_iter_session_fn);
}
EXPORT_SYMBOL_GPL(iscsi_host_for_each_session);

struct iscsi_scan_data {
        unsigned int channel;
        unsigned int id;
        u64 lun;
        enum scsi_scan_mode rescan;
};

static int iscsi_user_scan_session(struct device *dev, void *data)
{
        struct iscsi_scan_data *scan_data = data;
        struct iscsi_cls_session *session;
        struct Scsi_Host *shost;
        struct iscsi_cls_host *ihost;
        unsigned long flags;
        unsigned int id;

        if (!iscsi_is_session_dev(dev))
                return 0;

        session = iscsi_dev_to_session(dev);

        ISCSI_DBG_TRANS_SESSION(session, "Scanning session\n");

        shost = iscsi_session_to_shost(session);
        ihost = shost->shost_data;

        mutex_lock(&ihost->mutex);
        spin_lock_irqsave(&session->lock, flags);
        if (session->state != ISCSI_SESSION_LOGGED_IN) {
                spin_unlock_irqrestore(&session->lock, flags);
                goto user_scan_exit;
        }
        id = session->target_id;
        spin_unlock_irqrestore(&session->lock, flags);

        if (id != ISCSI_MAX_TARGET) {
                if ((scan_data->channel == SCAN_WILD_CARD ||
                     scan_data->channel == 0) &&
                    (scan_data->id == SCAN_WILD_CARD ||
                     scan_data->id == id)) {
                        scsi_scan_target(&session->dev, 0, id,
                                         scan_data->lun, scan_data->rescan);
                        spin_lock_irqsave(&session->lock, flags);
                        session->target_state = ISCSI_SESSION_TARGET_SCANNED;
                        spin_unlock_irqrestore(&session->lock, flags);
                }
        }

user_scan_exit:
        mutex_unlock(&ihost->mutex);
        ISCSI_DBG_TRANS_SESSION(session, "Completed session scan\n");
        return 0;
}

static int iscsi_user_scan(struct Scsi_Host *shost, uint channel,
                           uint id, u64 lun)
{
        struct iscsi_scan_data scan_data;

        scan_data.channel = channel;
        scan_data.id = id;
        scan_data.lun = lun;
        scan_data.rescan = SCSI_SCAN_MANUAL;

        return device_for_each_child(&shost->shost_gendev, &scan_data,
                                     iscsi_user_scan_session);
}

static void iscsi_scan_session(struct work_struct *work)
{
        struct iscsi_cls_session *session =
                        container_of(work, struct iscsi_cls_session, scan_work);
        struct iscsi_scan_data scan_data;

        scan_data.channel = 0;
        scan_data.id = SCAN_WILD_CARD;
        scan_data.lun = SCAN_WILD_CARD;
        scan_data.rescan = SCSI_SCAN_RESCAN;

        iscsi_user_scan_session(&session->dev, &scan_data);
}

/**
 * iscsi_block_scsi_eh - block scsi eh until session state has transistioned
 * @cmd: scsi cmd passed to scsi eh handler
 *
 * If the session is down this function will wait for the recovery
 * timer to fire or for the session to be logged back in. If the
 * recovery timer fires then FAST_IO_FAIL is returned. The caller
 * should pass this error value to the scsi eh.
 */
int iscsi_block_scsi_eh(struct scsi_cmnd *cmd)
{
        struct iscsi_cls_session *session =
                        starget_to_session(scsi_target(cmd->device));
        unsigned long flags;
        int ret = 0;

        spin_lock_irqsave(&session->lock, flags);
        while (session->state != ISCSI_SESSION_LOGGED_IN) {
                if (session->state == ISCSI_SESSION_FREE) {
                        ret = FAST_IO_FAIL;
                        break;
                }
                spin_unlock_irqrestore(&session->lock, flags);
                msleep(1000);
                spin_lock_irqsave(&session->lock, flags);
        }
        spin_unlock_irqrestore(&session->lock, flags);
        return ret;
}
EXPORT_SYMBOL_GPL(iscsi_block_scsi_eh);

static void session_recovery_timedout(struct work_struct *work)
{
        struct iscsi_cls_session *session =
                container_of(work, struct iscsi_cls_session,
                             recovery_work.work);
        unsigned long flags;

        iscsi_cls_session_printk(KERN_INFO, session,
                                 "session recovery timed out after %d secs\n",
                                 session->recovery_tmo);

        spin_lock_irqsave(&session->lock, flags);
        switch (session->state) {
        case ISCSI_SESSION_FAILED:
                session->state = ISCSI_SESSION_FREE;
                break;
        case ISCSI_SESSION_LOGGED_IN:
        case ISCSI_SESSION_FREE:
                /* we raced with the unblock's flush */
                spin_unlock_irqrestore(&session->lock, flags);
                return;
        }
        spin_unlock_irqrestore(&session->lock, flags);

        ISCSI_DBG_TRANS_SESSION(session, "Unblocking SCSI target\n");
        scsi_target_unblock(&session->dev, SDEV_TRANSPORT_OFFLINE);
        ISCSI_DBG_TRANS_SESSION(session, "Completed unblocking SCSI target\n");

        if (session->transport->session_recovery_timedout)
                session->transport->session_recovery_timedout(session);
}

static void __iscsi_unblock_session(struct work_struct *work)
{
        struct iscsi_cls_session *session =
                        container_of(work, struct iscsi_cls_session,
                                     unblock_work);
        unsigned long flags;

        ISCSI_DBG_TRANS_SESSION(session, "Unblocking session\n");

        cancel_delayed_work_sync(&session->recovery_work);
        spin_lock_irqsave(&session->lock, flags);
        session->state = ISCSI_SESSION_LOGGED_IN;
        spin_unlock_irqrestore(&session->lock, flags);
        /* start IO */
        scsi_target_unblock(&session->dev, SDEV_RUNNING);
        ISCSI_DBG_TRANS_SESSION(session, "Completed unblocking session\n");
}

/**
 * iscsi_unblock_session - set a session as logged in and start IO.
 * @session: iscsi session
 *
 * Mark a session as ready to accept IO.
 */
void iscsi_unblock_session(struct iscsi_cls_session *session)
{
        if (!cancel_work_sync(&session->block_work))
                cancel_delayed_work_sync(&session->recovery_work);

        queue_work(session->workq, &session->unblock_work);
        /*
         * Blocking the session can be done from any context so we only
         * queue the block work. Make sure the unblock work has completed
         * because it flushes/cancels the other works and updates the state.
         */
        flush_work(&session->unblock_work);
}
EXPORT_SYMBOL_GPL(iscsi_unblock_session);

static void __iscsi_block_session(struct work_struct *work)
{
        struct iscsi_cls_session *session =
                        container_of(work, struct iscsi_cls_session,
                                     block_work);
        struct Scsi_Host *shost = iscsi_session_to_shost(session);
        unsigned long flags;

        ISCSI_DBG_TRANS_SESSION(session, "Blocking session\n");
        spin_lock_irqsave(&session->lock, flags);
        session->state = ISCSI_SESSION_FAILED;
        spin_unlock_irqrestore(&session->lock, flags);
        scsi_block_targets(shost, &session->dev);
        ISCSI_DBG_TRANS_SESSION(session, "Completed SCSI target blocking\n");
        if (session->recovery_tmo >= 0)
                queue_delayed_work(session->workq,
                                   &session->recovery_work,
                                   session->recovery_tmo * HZ);
}

void iscsi_block_session(struct iscsi_cls_session *session)
{
        queue_work(session->workq, &session->block_work);
}
EXPORT_SYMBOL_GPL(iscsi_block_session);

static void __iscsi_unbind_session(struct work_struct *work)
{
        struct iscsi_cls_session *session =
                        container_of(work, struct iscsi_cls_session,
                                     unbind_work);
        struct Scsi_Host *shost = iscsi_session_to_shost(session);
        struct iscsi_cls_host *ihost = shost->shost_data;
        unsigned long flags;
        unsigned int target_id;
        bool remove_target = true;

        ISCSI_DBG_TRANS_SESSION(session, "Unbinding session\n");

        /* Prevent new scans and make sure scanning is not in progress */
        mutex_lock(&ihost->mutex);
        spin_lock_irqsave(&session->lock, flags);
        if (session->target_state == ISCSI_SESSION_TARGET_ALLOCATED) {
                remove_target = false;
        } else if (session->target_state != ISCSI_SESSION_TARGET_SCANNED) {
                spin_unlock_irqrestore(&session->lock, flags);
                mutex_unlock(&ihost->mutex);
                ISCSI_DBG_TRANS_SESSION(session,
                        "Skipping target unbinding: Session is unbound/unbinding.\n");
                return;
        }

        session->target_state = ISCSI_SESSION_TARGET_UNBINDING;
        target_id = session->target_id;
        session->target_id = ISCSI_MAX_TARGET;
        spin_unlock_irqrestore(&session->lock, flags);
        mutex_unlock(&ihost->mutex);

        if (remove_target)
                scsi_remove_target(&session->dev);

        if (session->ida_used)
                ida_free(&iscsi_sess_ida, target_id);

        iscsi_session_event(session, ISCSI_KEVENT_UNBIND_SESSION);
        ISCSI_DBG_TRANS_SESSION(session, "Completed target removal\n");

        spin_lock_irqsave(&session->lock, flags);
        session->target_state = ISCSI_SESSION_TARGET_UNBOUND;
        spin_unlock_irqrestore(&session->lock, flags);
}

static void __iscsi_destroy_session(struct work_struct *work)
{
        struct iscsi_cls_session *session =
                container_of(work, struct iscsi_cls_session, destroy_work);

        session->transport->destroy_session(session);
}

struct iscsi_cls_session *
iscsi_alloc_session(struct Scsi_Host *shost, struct iscsi_transport *transport,
                    int dd_size)
{
        struct iscsi_cls_session *session;

        session = kzalloc(sizeof(*session) + dd_size,
                          GFP_KERNEL);
        if (!session)
                return NULL;

        session->transport = transport;
        session->creator = -1;
        session->recovery_tmo = 120;
        session->recovery_tmo_sysfs_override = false;
        session->state = ISCSI_SESSION_FREE;
        INIT_DELAYED_WORK(&session->recovery_work, session_recovery_timedout);
        INIT_LIST_HEAD(&session->sess_list);
        INIT_WORK(&session->unblock_work, __iscsi_unblock_session);
        INIT_WORK(&session->block_work, __iscsi_block_session);
        INIT_WORK(&session->unbind_work, __iscsi_unbind_session);
        INIT_WORK(&session->scan_work, iscsi_scan_session);
        INIT_WORK(&session->destroy_work, __iscsi_destroy_session);
        spin_lock_init(&session->lock);

        /* this is released in the dev's release function */
        scsi_host_get(shost);
        session->dev.parent = &shost->shost_gendev;
        session->dev.release = iscsi_session_release;
        device_initialize(&session->dev);
        if (dd_size)
                session->dd_data = &session[1];

        ISCSI_DBG_TRANS_SESSION(session, "Completed session allocation\n");
        return session;
}
EXPORT_SYMBOL_GPL(iscsi_alloc_session);

int iscsi_add_session(struct iscsi_cls_session *session, unsigned int target_id)
{
        struct Scsi_Host *shost = iscsi_session_to_shost(session);
        unsigned long flags;
        int id = 0;
        int err;

        session->sid = atomic_add_return(1, &iscsi_session_nr);

        session->workq = alloc_workqueue("iscsi_ctrl_%d:%d",
                        WQ_SYSFS | WQ_MEM_RECLAIM | WQ_UNBOUND, 0,
                        shost->host_no, session->sid);
        if (!session->workq)
                return -ENOMEM;

        if (target_id == ISCSI_MAX_TARGET) {
                id = ida_alloc(&iscsi_sess_ida, GFP_KERNEL);

                if (id < 0) {
                        iscsi_cls_session_printk(KERN_ERR, session,
                                        "Failure in Target ID Allocation\n");
                        err = id;
                        goto destroy_wq;
                }
                session->target_id = (unsigned int)id;
                session->ida_used = true;
        } else
                session->target_id = target_id;
        spin_lock_irqsave(&session->lock, flags);
        session->target_state = ISCSI_SESSION_TARGET_ALLOCATED;
        spin_unlock_irqrestore(&session->lock, flags);

        dev_set_name(&session->dev, "session%u", session->sid);
        err = device_add(&session->dev);
        if (err) {
                iscsi_cls_session_printk(KERN_ERR, session,
                                         "could not register session's dev\n");
                goto release_ida;
        }
        err = transport_register_device(&session->dev);
        if (err) {
                iscsi_cls_session_printk(KERN_ERR, session,
                                         "could not register transport's dev\n");
                goto release_dev;
        }

        spin_lock_irqsave(&sesslock, flags);
        list_add(&session->sess_list, &sesslist);
        spin_unlock_irqrestore(&sesslock, flags);

        iscsi_session_event(session, ISCSI_KEVENT_CREATE_SESSION);
        ISCSI_DBG_TRANS_SESSION(session, "Completed session adding\n");
        return 0;

release_dev:
        device_del(&session->dev);
release_ida:
        if (session->ida_used)
                ida_free(&iscsi_sess_ida, session->target_id);
destroy_wq:
        destroy_workqueue(session->workq);
        return err;
}
EXPORT_SYMBOL_GPL(iscsi_add_session);

static void iscsi_conn_release(struct device *dev)
{
        struct iscsi_cls_conn *conn = iscsi_dev_to_conn(dev);
        struct device *parent = conn->dev.parent;

        ISCSI_DBG_TRANS_CONN(conn, "Releasing conn\n");
        kfree(conn);
        put_device(parent);
}

static int iscsi_is_conn_dev(const struct device *dev)
{
        return dev->release == iscsi_conn_release;
}

static int iscsi_iter_destroy_conn_fn(struct device *dev, void *data)
{
        if (!iscsi_is_conn_dev(dev))
                return 0;

        iscsi_remove_conn(iscsi_dev_to_conn(dev));
        iscsi_put_conn(iscsi_dev_to_conn(dev));

        return 0;
}

void iscsi_remove_session(struct iscsi_cls_session *session)
{
        unsigned long flags;
        int err;

        ISCSI_DBG_TRANS_SESSION(session, "Removing session\n");

        spin_lock_irqsave(&sesslock, flags);
        if (!list_empty(&session->sess_list))
                list_del(&session->sess_list);
        spin_unlock_irqrestore(&sesslock, flags);

        if (!cancel_work_sync(&session->block_work))
                cancel_delayed_work_sync(&session->recovery_work);
        cancel_work_sync(&session->unblock_work);
        /*
         * If we are blocked let commands flow again. The lld or iscsi
         * layer should set up the queuecommand to fail commands.
         * We assume that LLD will not be calling block/unblock while
         * removing the session.
         */
        spin_lock_irqsave(&session->lock, flags);
        session->state = ISCSI_SESSION_FREE;
        spin_unlock_irqrestore(&session->lock, flags);

        scsi_target_unblock(&session->dev, SDEV_TRANSPORT_OFFLINE);
        /*
         * qla4xxx can perform it's own scans when it runs in kernel only
         * mode. Make sure to flush those scans.
         */
        flush_work(&session->scan_work);
        /* flush running unbind operations */
        flush_work(&session->unbind_work);
        __iscsi_unbind_session(&session->unbind_work);

        /* hw iscsi may not have removed all connections from session */
        err = device_for_each_child(&session->dev, NULL,
                                    iscsi_iter_destroy_conn_fn);
        if (err)
                iscsi_cls_session_printk(KERN_ERR, session,
                                         "Could not delete all connections "
                                         "for session. Error %d.\n", err);

        transport_unregister_device(&session->dev);

        destroy_workqueue(session->workq);

        ISCSI_DBG_TRANS_SESSION(session, "Completing session removal\n");
        device_del(&session->dev);
}
EXPORT_SYMBOL_GPL(iscsi_remove_session);

static void iscsi_stop_conn(struct iscsi_cls_conn *conn, int flag)
{
        ISCSI_DBG_TRANS_CONN(conn, "Stopping conn.\n");

        switch (flag) {
        case STOP_CONN_RECOVER:
                WRITE_ONCE(conn->state, ISCSI_CONN_FAILED);
                break;
        case STOP_CONN_TERM:
                WRITE_ONCE(conn->state, ISCSI_CONN_DOWN);
                break;
        default:
                iscsi_cls_conn_printk(KERN_ERR, conn, "invalid stop flag %d\n",
                                      flag);
                return;
        }

        conn->transport->stop_conn(conn, flag);
        ISCSI_DBG_TRANS_CONN(conn, "Stopping conn done.\n");
}

static void iscsi_ep_disconnect(struct iscsi_cls_conn *conn, bool is_active)
{
        struct iscsi_cls_session *session = iscsi_conn_to_session(conn);
        struct iscsi_endpoint *ep;

        ISCSI_DBG_TRANS_CONN(conn, "disconnect ep.\n");
        WRITE_ONCE(conn->state, ISCSI_CONN_FAILED);

        if (!conn->ep || !session->transport->ep_disconnect)
                return;

        ep = conn->ep;
        conn->ep = NULL;

        session->transport->unbind_conn(conn, is_active);
        session->transport->ep_disconnect(ep);
        ISCSI_DBG_TRANS_CONN(conn, "disconnect ep done.\n");
}

static void iscsi_if_disconnect_bound_ep(struct iscsi_cls_conn *conn,
                                         struct iscsi_endpoint *ep,
                                         bool is_active)
{
        /* Check if this was a conn error and the kernel took ownership */
        spin_lock_irq(&conn->lock);
        if (!test_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags)) {
                spin_unlock_irq(&conn->lock);
                iscsi_ep_disconnect(conn, is_active);
        } else {
                spin_unlock_irq(&conn->lock);
                ISCSI_DBG_TRANS_CONN(conn, "flush kernel conn cleanup.\n");
                mutex_unlock(&conn->ep_mutex);

                flush_work(&conn->cleanup_work);
                /*
                 * Userspace is now done with the EP so we can release the ref
                 * iscsi_cleanup_conn_work_fn took.
                 */
                iscsi_put_endpoint(ep);
                mutex_lock(&conn->ep_mutex);
        }
}

static int iscsi_if_stop_conn(struct iscsi_cls_conn *conn, int flag)
{
        ISCSI_DBG_TRANS_CONN(conn, "iscsi if conn stop.\n");
        /*
         * For offload, iscsid may not know about the ep like when iscsid is
         * restarted or for kernel based session shutdown iscsid is not even
         * up. For these cases, we do the disconnect now.
         */
        mutex_lock(&conn->ep_mutex);
        if (conn->ep)
                iscsi_if_disconnect_bound_ep(conn, conn->ep, true);
        mutex_unlock(&conn->ep_mutex);

        /*
         * If this is a termination we have to call stop_conn with that flag
         * so the correct states get set. If we haven't run the work yet try to
         * avoid the extra run.
         */
        if (flag == STOP_CONN_TERM) {
                cancel_work_sync(&conn->cleanup_work);
                iscsi_stop_conn(conn, flag);
        } else {
                /*
                 * Figure out if it was the kernel or userspace initiating this.
                 */
                spin_lock_irq(&conn->lock);
                if (!test_and_set_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags)) {
                        spin_unlock_irq(&conn->lock);
                        iscsi_stop_conn(conn, flag);
                } else {
                        spin_unlock_irq(&conn->lock);
                        ISCSI_DBG_TRANS_CONN(conn,
                                             "flush kernel conn cleanup.\n");
                        flush_work(&conn->cleanup_work);
                }
                /*
                 * Only clear for recovery to avoid extra cleanup runs during
                 * termination.
                 */
                spin_lock_irq(&conn->lock);
                clear_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags);
                spin_unlock_irq(&conn->lock);
        }
        ISCSI_DBG_TRANS_CONN(conn, "iscsi if conn stop done.\n");
        return 0;
}

static void iscsi_cleanup_conn_work_fn(struct work_struct *work)
{
        struct iscsi_cls_conn *conn = container_of(work, struct iscsi_cls_conn,
                                                   cleanup_work);
        struct iscsi_cls_session *session = iscsi_conn_to_session(conn);

        mutex_lock(&conn->ep_mutex);
        /*
         * Get a ref to the ep, so we don't release its ID until after
         * userspace is done referencing it in iscsi_if_disconnect_bound_ep.
         */
        if (conn->ep)
                get_device(&conn->ep->dev);
        iscsi_ep_disconnect(conn, false);

        if (system_state != SYSTEM_RUNNING) {
                /*
                 * If the user has set up for the session to never timeout
                 * then hang like they wanted. For all other cases fail right
                 * away since userspace is not going to relogin.
                 */
                if (session->recovery_tmo > 0)
                        session->recovery_tmo = 0;
        }

        iscsi_stop_conn(conn, STOP_CONN_RECOVER);
        mutex_unlock(&conn->ep_mutex);
        ISCSI_DBG_TRANS_CONN(conn, "cleanup done.\n");
}

static int iscsi_iter_force_destroy_conn_fn(struct device *dev, void *data)
{
        struct iscsi_transport *transport;
        struct iscsi_cls_conn *conn;

        if (!iscsi_is_conn_dev(dev))
                return 0;

        conn = iscsi_dev_to_conn(dev);
        transport = conn->transport;

        if (READ_ONCE(conn->state) != ISCSI_CONN_DOWN)
                iscsi_if_stop_conn(conn, STOP_CONN_TERM);

        transport->destroy_conn(conn);
        return 0;
}

/**
 * iscsi_force_destroy_session - destroy a session from the kernel
 * @session: session to destroy
 *
 * Force the destruction of a session from the kernel. This should only be
 * used when userspace is no longer running during system shutdown.
 */
void iscsi_force_destroy_session(struct iscsi_cls_session *session)
{
        struct iscsi_transport *transport = session->transport;
        unsigned long flags;

        WARN_ON_ONCE(system_state == SYSTEM_RUNNING);

        spin_lock_irqsave(&sesslock, flags);
        if (list_empty(&session->sess_list)) {
                spin_unlock_irqrestore(&sesslock, flags);
                /*
                 * Conn/ep is already freed. Session is being torn down via
                 * async path. For shutdown we don't care about it so return.
                 */
                return;
        }
        spin_unlock_irqrestore(&sesslock, flags);

        device_for_each_child(&session->dev, NULL,
                              iscsi_iter_force_destroy_conn_fn);
        transport->destroy_session(session);
}
EXPORT_SYMBOL_GPL(iscsi_force_destroy_session);

void iscsi_free_session(struct iscsi_cls_session *session)
{
        ISCSI_DBG_TRANS_SESSION(session, "Freeing session\n");
        iscsi_session_event(session, ISCSI_KEVENT_DESTROY_SESSION);
        put_device(&session->dev);
}
EXPORT_SYMBOL_GPL(iscsi_free_session);

/**
 * iscsi_alloc_conn - alloc iscsi class connection
 * @session: iscsi cls session
 * @dd_size: private driver data size
 * @cid: connection id
 */
struct iscsi_cls_conn *
iscsi_alloc_conn(struct iscsi_cls_session *session, int dd_size, uint32_t cid)
{
        struct iscsi_transport *transport = session->transport;
        struct iscsi_cls_conn *conn;

        conn = kzalloc(sizeof(*conn) + dd_size, GFP_KERNEL);
        if (!conn)
                return NULL;
        if (dd_size)
                conn->dd_data = &conn[1];

        mutex_init(&conn->ep_mutex);
        spin_lock_init(&conn->lock);
        INIT_LIST_HEAD(&conn->conn_list);
        INIT_WORK(&conn->cleanup_work, iscsi_cleanup_conn_work_fn);
        conn->transport = transport;
        conn->cid = cid;
        WRITE_ONCE(conn->state, ISCSI_CONN_DOWN);

        /* this is released in the dev's release function */
        if (!get_device(&session->dev))
                goto free_conn;

        dev_set_name(&conn->dev, "connection%d:%u", session->sid, cid);
        device_initialize(&conn->dev);
        conn->dev.parent = &session->dev;
        conn->dev.release = iscsi_conn_release;

        return conn;

free_conn:
        kfree(conn);
        return NULL;
}
EXPORT_SYMBOL_GPL(iscsi_alloc_conn);

/**
 * iscsi_add_conn - add iscsi class connection
 * @conn: iscsi cls connection
 *
 * This will expose iscsi_cls_conn to sysfs so make sure the related
 * resources for sysfs attributes are initialized before calling this.
 */
int iscsi_add_conn(struct iscsi_cls_conn *conn)
{
        int err;
        unsigned long flags;
        struct iscsi_cls_session *session = iscsi_dev_to_session(conn->dev.parent);

        err = device_add(&conn->dev);
        if (err) {
                iscsi_cls_session_printk(KERN_ERR, session,
                                         "could not register connection's dev\n");
                return err;
        }
        err = transport_register_device(&conn->dev);
        if (err) {
                iscsi_cls_session_printk(KERN_ERR, session,
                                         "could not register transport's dev\n");
                device_del(&conn->dev);
                return err;
        }

        spin_lock_irqsave(&connlock, flags);
        list_add(&conn->conn_list, &connlist);
        spin_unlock_irqrestore(&connlock, flags);

        return 0;
}
EXPORT_SYMBOL_GPL(iscsi_add_conn);

/**
 * iscsi_remove_conn - remove iscsi class connection from sysfs
 * @conn: iscsi cls connection
 *
 * Remove iscsi_cls_conn from sysfs, and wait for previous
 * read/write of iscsi_cls_conn's attributes in sysfs to finish.
 */
void iscsi_remove_conn(struct iscsi_cls_conn *conn)
{
        unsigned long flags;

        spin_lock_irqsave(&connlock, flags);
        list_del(&conn->conn_list);
        spin_unlock_irqrestore(&connlock, flags);

        transport_unregister_device(&conn->dev);
        device_del(&conn->dev);
}
EXPORT_SYMBOL_GPL(iscsi_remove_conn);

void iscsi_put_conn(struct iscsi_cls_conn *conn)
{
        put_device(&conn->dev);
}
EXPORT_SYMBOL_GPL(iscsi_put_conn);

void iscsi_get_conn(struct iscsi_cls_conn *conn)
{
        get_device(&conn->dev);
}
EXPORT_SYMBOL_GPL(iscsi_get_conn);

/*
 * iscsi interface functions
 */
static struct iscsi_internal *
iscsi_if_transport_lookup(struct iscsi_transport *tt)
{
        struct iscsi_internal *priv;
        unsigned long flags;

        spin_lock_irqsave(&iscsi_transport_lock, flags);
        list_for_each_entry(priv, &iscsi_transports, list) {
                if (tt == priv->iscsi_transport) {
                        spin_unlock_irqrestore(&iscsi_transport_lock, flags);
                        return priv;
                }
        }
        spin_unlock_irqrestore(&iscsi_transport_lock, flags);
        return NULL;
}

static int
iscsi_multicast_skb(struct sk_buff *skb, uint32_t group, gfp_t gfp)
{
        return nlmsg_multicast(nls, skb, 0, group, gfp);
}

static int
iscsi_unicast_skb(struct sk_buff *skb, u32 portid)
{
        return nlmsg_unicast(nls, skb, portid);
}

int iscsi_recv_pdu(struct iscsi_cls_conn *conn, struct iscsi_hdr *hdr,
                   char *data, uint32_t data_size)
{
        struct nlmsghdr        *nlh;
        struct sk_buff *skb;
        struct iscsi_uevent *ev;
        char *pdu;
        struct iscsi_internal *priv;
        int len = nlmsg_total_size(sizeof(*ev) + sizeof(struct iscsi_hdr) +
                                   data_size);

        priv = iscsi_if_transport_lookup(conn->transport);
        if (!priv)
                return -EINVAL;

        skb = alloc_skb(len, GFP_ATOMIC);
        if (!skb) {
                iscsi_conn_error_event(conn, ISCSI_ERR_CONN_FAILED);
                iscsi_cls_conn_printk(KERN_ERR, conn, "can not deliver "
                                      "control PDU: OOM\n");
                return -ENOMEM;
        }

        nlh = __nlmsg_put(skb, 0, 0, 0, (len - sizeof(*nlh)), 0);
        ev = nlmsg_data(nlh);
        memset(ev, 0, sizeof(*ev));
        ev->transport_handle = iscsi_handle(conn->transport);
        ev->type = ISCSI_KEVENT_RECV_PDU;
        ev->r.recv_req.cid = conn->cid;
        ev->r.recv_req.sid = iscsi_conn_get_sid(conn);
        pdu = (char*)ev + sizeof(*ev);
        memcpy(pdu, hdr, sizeof(struct iscsi_hdr));
        memcpy(pdu + sizeof(struct iscsi_hdr), data, data_size);

        return iscsi_multicast_skb(skb, ISCSI_NL_GRP_ISCSID, GFP_ATOMIC);
}
EXPORT_SYMBOL_GPL(iscsi_recv_pdu);

int iscsi_offload_mesg(struct Scsi_Host *shost,
                       struct iscsi_transport *transport, uint32_t type,
                       char *data, uint16_t data_size)
{
        struct nlmsghdr        *nlh;
        struct sk_buff *skb;
        struct iscsi_uevent *ev;
        int len = nlmsg_total_size(sizeof(*ev) + data_size);

        skb = alloc_skb(len, GFP_ATOMIC);
        if (!skb) {
                printk(KERN_ERR "can not deliver iscsi offload message:OOM\n");
                return -ENOMEM;
        }

        nlh = __nlmsg_put(skb, 0, 0, 0, (len - sizeof(*nlh)), 0);
        ev = nlmsg_data(nlh);
        memset(ev, 0, sizeof(*ev));
        ev->type = type;
        ev->transport_handle = iscsi_handle(transport);
        switch (type) {
        case ISCSI_KEVENT_PATH_REQ:
                ev->r.req_path.host_no = shost->host_no;
                break;
        case ISCSI_KEVENT_IF_DOWN:
                ev->r.notify_if_down.host_no = shost->host_no;
                break;
        }

        memcpy((char *)ev + sizeof(*ev), data, data_size);

        return iscsi_multicast_skb(skb, ISCSI_NL_GRP_UIP, GFP_ATOMIC);
}
EXPORT_SYMBOL_GPL(iscsi_offload_mesg);

void iscsi_conn_error_event(struct iscsi_cls_conn *conn, enum iscsi_err error)
{
        struct nlmsghdr        *nlh;
        struct sk_buff        *skb;
        struct iscsi_uevent *ev;
        struct iscsi_internal *priv;
        int len = nlmsg_total_size(sizeof(*ev));
        unsigned long flags;
        int state;

        spin_lock_irqsave(&conn->lock, flags);
        /*
         * Userspace will only do a stop call if we are at least bound. And, we
         * only need to do the in kernel cleanup if in the UP state so cmds can
         * be released to upper layers. If in other states just wait for
         * userspace to avoid races that can leave the cleanup_work queued.
         */
        state = READ_ONCE(conn->state);
        switch (state) {
        case ISCSI_CONN_BOUND:
        case ISCSI_CONN_UP:
                if (!test_and_set_bit(ISCSI_CLS_CONN_BIT_CLEANUP,
                                      &conn->flags)) {
                        queue_work(iscsi_conn_cleanup_workq,
                                   &conn->cleanup_work);
                }
                break;
        default:
                ISCSI_DBG_TRANS_CONN(conn, "Got conn error in state %d\n",
                                     state);
                break;
        }
        spin_unlock_irqrestore(&conn->lock, flags);

        priv = iscsi_if_transport_lookup(conn->transport);
        if (!priv)
                return;

        skb = alloc_skb(len, GFP_ATOMIC);
        if (!skb) {
                iscsi_cls_conn_printk(KERN_ERR, conn, "gracefully ignored "
                                      "conn error (%d)\n", error);
                return;
        }

        nlh = __nlmsg_put(skb, 0, 0, 0, (len - sizeof(*nlh)), 0);
        ev = nlmsg_data(nlh);
        ev->transport_handle = iscsi_handle(conn->transport);
        ev->type = ISCSI_KEVENT_CONN_ERROR;
        ev->r.connerror.error = error;
        ev->r.connerror.cid = conn->cid;
        ev->r.connerror.sid = iscsi_conn_get_sid(conn);

        iscsi_multicast_skb(skb, ISCSI_NL_GRP_ISCSID, GFP_ATOMIC);

        iscsi_cls_conn_printk(KERN_INFO, conn, "detected conn error (%d)\n",
                              error);
}
EXPORT_SYMBOL_GPL(iscsi_conn_error_event);

void iscsi_conn_login_event(struct iscsi_cls_conn *conn,
                            enum iscsi_conn_state state)
{
        struct nlmsghdr *nlh;
        struct sk_buff  *skb;
        struct iscsi_uevent *ev;
        struct iscsi_internal *priv;
        int len = nlmsg_total_size(sizeof(*ev));

        priv = iscsi_if_transport_lookup(conn->transport);
        if (!priv)
                return;

        skb = alloc_skb(len, GFP_ATOMIC);
        if (!skb) {
                iscsi_cls_conn_printk(KERN_ERR, conn, "gracefully ignored "
                                      "conn login (%d)\n", state);
                return;
        }

        nlh = __nlmsg_put(skb, 0, 0, 0, (len - sizeof(*nlh)), 0);
        ev = nlmsg_data(nlh);
        ev->transport_handle = iscsi_handle(conn->transport);
        ev->type = ISCSI_KEVENT_CONN_LOGIN_STATE;
        ev->r.conn_login.state = state;
        ev->r.conn_login.cid = conn->cid;
        ev->r.conn_login.sid = iscsi_conn_get_sid(conn);
        iscsi_multicast_skb(skb, ISCSI_NL_GRP_ISCSID, GFP_ATOMIC);

        iscsi_cls_conn_printk(KERN_INFO, conn, "detected conn login (%d)\n",
                              state);
}
EXPORT_SYMBOL_GPL(iscsi_conn_login_event);

void iscsi_post_host_event(uint32_t host_no, struct iscsi_transport *transport,
                           enum iscsi_host_event_code code, uint32_t data_size,
                           uint8_t *data)
{
        struct nlmsghdr *nlh;
        struct sk_buff *skb;
        struct iscsi_uevent *ev;
        int len = nlmsg_total_size(sizeof(*ev) + data_size);

        skb = alloc_skb(len, GFP_NOIO);
        if (!skb) {
                printk(KERN_ERR "gracefully ignored host event (%d):%d OOM\n",
                       host_no, code);
                return;
        }

        nlh = __nlmsg_put(skb, 0, 0, 0, (len - sizeof(*nlh)), 0);
        ev = nlmsg_data(nlh);
        ev->transport_handle = iscsi_handle(transport);
        ev->type = ISCSI_KEVENT_HOST_EVENT;
        ev->r.host_event.host_no = host_no;
        ev->r.host_event.code = code;
        ev->r.host_event.data_size = data_size;

        if (data_size)
                memcpy((char *)ev + sizeof(*ev), data, data_size);

        iscsi_multicast_skb(skb, ISCSI_NL_GRP_ISCSID, GFP_NOIO);
}
EXPORT_SYMBOL_GPL(iscsi_post_host_event);

void iscsi_ping_comp_event(uint32_t host_no, struct iscsi_transport *transport,
                           uint32_t status, uint32_t pid, uint32_t data_size,
                           uint8_t *data)
{
        struct nlmsghdr *nlh;
        struct sk_buff *skb;
        struct iscsi_uevent *ev;
        int len = nlmsg_total_size(sizeof(*ev) + data_size);

        skb = alloc_skb(len, GFP_NOIO);
        if (!skb) {
                printk(KERN_ERR "gracefully ignored ping comp: OOM\n");
                return;
        }

        nlh = __nlmsg_put(skb, 0, 0, 0, (len - sizeof(*nlh)), 0);
        ev = nlmsg_data(nlh);
        ev->transport_handle = iscsi_handle(transport);
        ev->type = ISCSI_KEVENT_PING_COMP;
        ev->r.ping_comp.host_no = host_no;
        ev->r.ping_comp.status = status;
        ev->r.ping_comp.pid = pid;
        ev->r.ping_comp.data_size = data_size;
        memcpy((char *)ev + sizeof(*ev), data, data_size);

        iscsi_multicast_skb(skb, ISCSI_NL_GRP_ISCSID, GFP_NOIO);
}
EXPORT_SYMBOL_GPL(iscsi_ping_comp_event);

static int
iscsi_if_send_reply(u32 portid, int type, void *payload, int size)
{
        struct sk_buff        *skb;
        struct nlmsghdr        *nlh;
        int len = nlmsg_total_size(size);

        skb = alloc_skb(len, GFP_ATOMIC);
        if (!skb) {
                printk(KERN_ERR "Could not allocate skb to send reply.\n");
                return -ENOMEM;
        }

        nlh = __nlmsg_put(skb, 0, 0, type, (len - sizeof(*nlh)), 0);
        memcpy(nlmsg_data(nlh), payload, size);
        return iscsi_unicast_skb(skb, portid);
}

static int
iscsi_if_get_stats(struct iscsi_transport *transport, struct nlmsghdr *nlh)
{
        struct iscsi_uevent *ev = nlmsg_data(nlh);
        struct iscsi_stats *stats;
        struct sk_buff *skbstat;
        struct iscsi_cls_conn *conn;
        struct nlmsghdr        *nlhstat;
        struct iscsi_uevent *evstat;
        struct iscsi_internal *priv;
        int len = nlmsg_total_size(sizeof(*ev) +
                                   sizeof(struct iscsi_stats) +
                                   sizeof(struct iscsi_stats_custom) *
                                   ISCSI_STATS_CUSTOM_MAX);
        int err = 0;

        priv = iscsi_if_transport_lookup(transport);
        if (!priv)
                return -EINVAL;

        conn = iscsi_conn_lookup(ev->u.get_stats.sid, ev->u.get_stats.cid);
        if (!conn)
                return -EEXIST;

        do {
                int actual_size;

                skbstat = alloc_skb(len, GFP_ATOMIC);
                if (!skbstat) {
                        iscsi_cls_conn_printk(KERN_ERR, conn, "can not "
                                              "deliver stats: OOM\n");
                        return -ENOMEM;
                }

                nlhstat = __nlmsg_put(skbstat, 0, 0, 0,
                                      (len - sizeof(*nlhstat)), 0);
                evstat = nlmsg_data(nlhstat);
                memset(evstat, 0, sizeof(*evstat));
                evstat->transport_handle = iscsi_handle(conn->transport);
                evstat->type = nlh->nlmsg_type;
                evstat->u.get_stats.cid =
                        ev->u.get_stats.cid;
                evstat->u.get_stats.sid =
                        ev->u.get_stats.sid;
                stats = (struct iscsi_stats *)
                        ((char*)evstat + sizeof(*evstat));
                memset(stats, 0, sizeof(*stats));

                transport->get_stats(conn, stats);
                actual_size = nlmsg_total_size(sizeof(struct iscsi_uevent) +
                                               sizeof(struct iscsi_stats) +
                                               sizeof(struct iscsi_stats_custom) *
                                               stats->custom_length);
                actual_size -= sizeof(*nlhstat);
                actual_size = nlmsg_msg_size(actual_size);
                skb_trim(skbstat, NLMSG_ALIGN(actual_size));
                nlhstat->nlmsg_len = actual_size;

                err = iscsi_multicast_skb(skbstat, ISCSI_NL_GRP_ISCSID,
                                          GFP_ATOMIC);
        } while (err < 0 && err != -ECONNREFUSED);

        return err;
}

/**
 * iscsi_session_event - send session destr. completion event
 * @session: iscsi class session
 * @event: type of event
 */
int iscsi_session_event(struct iscsi_cls_session *session,
                        enum iscsi_uevent_e event)
{
        struct iscsi_internal *priv;
        struct Scsi_Host *shost;
        struct iscsi_uevent *ev;
        struct sk_buff  *skb;
        struct nlmsghdr *nlh;
        int rc, len = nlmsg_total_size(sizeof(*ev));

        priv = iscsi_if_transport_lookup(session->transport);
        if (!priv)
                return -EINVAL;
        shost = iscsi_session_to_shost(session);

        skb = alloc_skb(len, GFP_KERNEL);
        if (!skb) {
                iscsi_cls_session_printk(KERN_ERR, session,
                                         "Cannot notify userspace of session "
                                         "event %u\n", event);
                return -ENOMEM;
        }

        nlh = __nlmsg_put(skb, 0, 0, 0, (len - sizeof(*nlh)), 0);
        ev = nlmsg_data(nlh);
        ev->transport_handle = iscsi_handle(session->transport);

        ev->type = event;
        switch (event) {
        case ISCSI_KEVENT_DESTROY_SESSION:
                ev->r.d_session.host_no = shost->host_no;
                ev->r.d_session.sid = session->sid;
                break;
        case ISCSI_KEVENT_CREATE_SESSION:
                ev->r.c_session_ret.host_no = shost->host_no;
                ev->r.c_session_ret.sid = session->sid;
                break;
        case ISCSI_KEVENT_UNBIND_SESSION:
                ev->r.unbind_session.host_no = shost->host_no;
                ev->r.unbind_session.sid = session->sid;
                break;
        default:
                iscsi_cls_session_printk(KERN_ERR, session, "Invalid event "
                                         "%u.\n", event);
                kfree_skb(skb);
                return -EINVAL;
        }

        /*
         * this will occur if the daemon is not up, so we just warn
         * the user and when the daemon is restarted it will handle it
         */
        rc = iscsi_multicast_skb(skb, ISCSI_NL_GRP_ISCSID, GFP_KERNEL);
        if (rc == -ESRCH)
                iscsi_cls_session_printk(KERN_ERR, session,
                                         "Cannot notify userspace of session "
                                         "event %u. Check iscsi daemon\n",
                                         event);

        ISCSI_DBG_TRANS_SESSION(session, "Completed handling event %d rc %d\n",
                                event, rc);
        return rc;
}
EXPORT_SYMBOL_GPL(iscsi_session_event);

static int
iscsi_if_create_session(struct iscsi_internal *priv, struct iscsi_endpoint *ep,
                        struct iscsi_uevent *ev, pid_t pid,
                        uint32_t initial_cmdsn,        uint16_t cmds_max,
                        uint16_t queue_depth)
{
        struct iscsi_transport *transport = priv->iscsi_transport;
        struct iscsi_cls_session *session;
        struct Scsi_Host *shost;

        session = transport->create_session(ep, cmds_max, queue_depth,
                                            initial_cmdsn);
        if (!session)
                return -ENOMEM;

        session->creator = pid;
        shost = iscsi_session_to_shost(session);
        ev->r.c_session_ret.host_no = shost->host_no;
        ev->r.c_session_ret.sid = session->sid;
        ISCSI_DBG_TRANS_SESSION(session,
                                "Completed creating transport session\n");
        return 0;
}

static int
iscsi_if_create_conn(struct iscsi_transport *transport, struct iscsi_uevent *ev)
{
        struct iscsi_cls_conn *conn;
        struct iscsi_cls_session *session;

        session = iscsi_session_lookup(ev->u.c_conn.sid);
        if (!session) {
                printk(KERN_ERR "iscsi: invalid session %d.\n",
                       ev->u.c_conn.sid);
                return -EINVAL;
        }

        conn = transport->create_conn(session, ev->u.c_conn.cid);
        if (!conn) {
                iscsi_cls_session_printk(KERN_ERR, session,
                                         "couldn't create a new connection.");
                return -ENOMEM;
        }

        ev->r.c_conn_ret.sid = session->sid;
        ev->r.c_conn_ret.cid = conn->cid;

        ISCSI_DBG_TRANS_CONN(conn, "Completed creating transport conn\n");
        return 0;
}

static int
iscsi_if_destroy_conn(struct iscsi_transport *transport, struct iscsi_uevent *ev)
{
        struct iscsi_cls_conn *conn;

        conn = iscsi_conn_lookup(ev->u.d_conn.sid, ev->u.d_conn.cid);
        if (!conn)
                return -EINVAL;

        ISCSI_DBG_TRANS_CONN(conn, "Flushing cleanup during destruction\n");
        flush_work(&conn->cleanup_work);
        ISCSI_DBG_TRANS_CONN(conn, "Destroying transport conn\n");

        if (transport->destroy_conn)
                transport->destroy_conn(conn);
        return 0;
}

static int
iscsi_if_set_param(struct iscsi_transport *transport, struct iscsi_uevent *ev, u32 rlen)
{
        char *data = (char*)ev + sizeof(*ev);
        struct iscsi_cls_conn *conn;
        struct iscsi_cls_session *session;
        int err = 0, value = 0, state;

        if (ev->u.set_param.len > rlen ||
            ev->u.set_param.len > PAGE_SIZE)
                return -EINVAL;

        session = iscsi_session_lookup(ev->u.set_param.sid);
        conn = iscsi_conn_lookup(ev->u.set_param.sid, ev->u.set_param.cid);
        if (!conn || !session)
                return -EINVAL;

        /* data will be regarded as NULL-ended string, do length check */
        if (strlen(data) > ev->u.set_param.len)
                return -EINVAL;

        switch (ev->u.set_param.param) {
        case ISCSI_PARAM_SESS_RECOVERY_TMO:
                sscanf(data, "%d", &value);
                if (!session->recovery_tmo_sysfs_override)
                        session->recovery_tmo = value;
                break;
        default:
                state = READ_ONCE(conn->state);
                if (state == ISCSI_CONN_BOUND || state == ISCSI_CONN_UP) {
                        err = transport->set_param(conn, ev->u.set_param.param,
                                        data, ev->u.set_param.len);
                } else {
                        return -ENOTCONN;
                }
        }

        return err;
}

static int iscsi_if_ep_connect(struct iscsi_transport *transport,
                               struct iscsi_uevent *ev, int msg_type)
{
        struct iscsi_endpoint *ep;
        struct sockaddr *dst_addr;
        struct Scsi_Host *shost = NULL;
        int non_blocking, err = 0;

        if (!transport->ep_connect)
                return -EINVAL;

        if (msg_type == ISCSI_UEVENT_TRANSPORT_EP_CONNECT_THROUGH_HOST) {
                shost = scsi_host_lookup(ev->u.ep_connect_through_host.host_no);
                if (!shost) {
                        printk(KERN_ERR "ep connect failed. Could not find "
                               "host no %u\n",
                               ev->u.ep_connect_through_host.host_no);
                        return -ENODEV;
                }
                non_blocking = ev->u.ep_connect_through_host.non_blocking;
        } else
                non_blocking = ev->u.ep_connect.non_blocking;

        dst_addr = (struct sockaddr *)((char*)ev + sizeof(*ev));
        ep = transport->ep_connect(shost, dst_addr, non_blocking);
        if (IS_ERR(ep)) {
                err = PTR_ERR(ep);
                goto release_host;
        }

        ev->r.ep_connect_ret.handle = ep->id;
release_host:
        if (shost)
                scsi_host_put(shost);
        return err;
}

static int iscsi_if_ep_disconnect(struct iscsi_transport *transport,
                                  u64 ep_handle)
{
        struct iscsi_cls_conn *conn;
        struct iscsi_endpoint *ep;

        if (!transport->ep_disconnect)
                return -EINVAL;

        ep = iscsi_lookup_endpoint(ep_handle);
        if (!ep)
                return -EINVAL;

        conn = ep->conn;
        if (!conn) {
                /*
                 * conn was not even bound yet, so we can't get iscsi conn
                 * failures yet.
                 */
                transport->ep_disconnect(ep);
                goto put_ep;
        }

        mutex_lock(&conn->ep_mutex);
        iscsi_if_disconnect_bound_ep(conn, ep, false);
        mutex_unlock(&conn->ep_mutex);
put_ep:
        iscsi_put_endpoint(ep);
        return 0;
}

static int
iscsi_if_transport_ep(struct iscsi_transport *transport,
                      struct iscsi_uevent *ev, int msg_type, u32 rlen)
{
        struct iscsi_endpoint *ep;
        int rc = 0;

        switch (msg_type) {
        case ISCSI_UEVENT_TRANSPORT_EP_CONNECT_THROUGH_HOST:
        case ISCSI_UEVENT_TRANSPORT_EP_CONNECT:
                if (rlen < sizeof(struct sockaddr))
                        rc = -EINVAL;
                else
                        rc = iscsi_if_ep_connect(transport, ev, msg_type);
                break;
        case ISCSI_UEVENT_TRANSPORT_EP_POLL:
                if (!transport->ep_poll)
                        return -EINVAL;

                ep = iscsi_lookup_endpoint(ev->u.ep_poll.ep_handle);
                if (!ep)
                        return -EINVAL;

                ev->r.retcode = transport->ep_poll(ep,
                                                   ev->u.ep_poll.timeout_ms);
                iscsi_put_endpoint(ep);
                break;
        case ISCSI_UEVENT_TRANSPORT_EP_DISCONNECT:
                rc = iscsi_if_ep_disconnect(transport,
                                            ev->u.ep_disconnect.ep_handle);
                break;
        }
        return rc;
}

static int
iscsi_tgt_dscvr(struct iscsi_transport *transport,
                struct iscsi_uevent *ev, u32 rlen)
{
        struct Scsi_Host *shost;
        struct sockaddr *dst_addr;
        int err;

        if (rlen < sizeof(*dst_addr))
                return -EINVAL;

        if (!transport->tgt_dscvr)
                return -EINVAL;

        shost = scsi_host_lookup(ev->u.tgt_dscvr.host_no);
        if (!shost) {
                printk(KERN_ERR "target discovery could not find host no %u\n",
                       ev->u.tgt_dscvr.host_no);
                return -ENODEV;
        }


        dst_addr = (struct sockaddr *)((char*)ev + sizeof(*ev));
        err = transport->tgt_dscvr(shost, ev->u.tgt_dscvr.type,
                                   ev->u.tgt_dscvr.enable, dst_addr);
        scsi_host_put(shost);
        return err;
}

static int
iscsi_set_host_param(struct iscsi_transport *transport,
                     struct iscsi_uevent *ev, u32 rlen)
{
        char *data = (char*)ev + sizeof(*ev);
        struct Scsi_Host *shost;
        int err;

        if (!transport->set_host_param)
                return -ENOSYS;

        if (ev->u.set_host_param.len > rlen ||
            ev->u.set_host_param.len > PAGE_SIZE)
                return -EINVAL;

        shost = scsi_host_lookup(ev->u.set_host_param.host_no);
        if (!shost) {
                printk(KERN_ERR "set_host_param could not find host no %u\n",
                       ev->u.set_host_param.host_no);
                return -ENODEV;
        }

        /* see similar check in iscsi_if_set_param() */
        if (strlen(data) > ev->u.set_host_param.len) {
                err = -EINVAL;
                goto out;
        }

        err = transport->set_host_param(shost, ev->u.set_host_param.param,
                                        data, ev->u.set_host_param.len);
out:
        scsi_host_put(shost);
        return err;
}

static int
iscsi_set_path(struct iscsi_transport *transport, struct iscsi_uevent *ev, u32 rlen)
{
        struct Scsi_Host *shost;
        struct iscsi_path *params;
        int err;

        if (rlen < sizeof(*params))
                return -EINVAL;

        if (!transport->set_path)
                return -ENOSYS;

        shost = scsi_host_lookup(ev->u.set_path.host_no);
        if (!shost) {
                printk(KERN_ERR "set path could not find host no %u\n",
                       ev->u.set_path.host_no);
                return -ENODEV;
        }

        params = (struct iscsi_path *)((char *)ev + sizeof(*ev));
        err = transport->set_path(shost, params);

        scsi_host_put(shost);
        return err;
}

static int iscsi_session_has_conns(int sid)
{
        struct iscsi_cls_conn *conn;
        unsigned long flags;
        int found = 0;

        spin_lock_irqsave(&connlock, flags);
        list_for_each_entry(conn, &connlist, conn_list) {
                if (iscsi_conn_get_sid(conn) == sid) {
                        found = 1;
                        break;
                }
        }
        spin_unlock_irqrestore(&connlock, flags);

        return found;
}

static int
iscsi_set_iface_params(struct iscsi_transport *transport,
                       struct iscsi_uevent *ev, uint32_t len)
{
        char *data = (char *)ev + sizeof(*ev);
        struct Scsi_Host *shost;
        int err;

        if (!transport->set_iface_param)
                return -ENOSYS;

        shost = scsi_host_lookup(ev->u.set_iface_params.host_no);
        if (!shost) {
                printk(KERN_ERR "set_iface_params could not find host no %u\n",
                       ev->u.set_iface_params.host_no);
                return -ENODEV;
        }

        err = transport->set_iface_param(shost, data, len);
        scsi_host_put(shost);
        return err;
}

static int
iscsi_send_ping(struct iscsi_transport *transport, struct iscsi_uevent *ev, u32 rlen)
{
        struct Scsi_Host *shost;
        struct sockaddr *dst_addr;
        int err;

        if (rlen < sizeof(*dst_addr))
                return -EINVAL;

        if (!transport->send_ping)
                return -ENOSYS;

        shost = scsi_host_lookup(ev->u.iscsi_ping.host_no);
        if (!shost) {
                printk(KERN_ERR "iscsi_ping could not find host no %u\n",
                       ev->u.iscsi_ping.host_no);
                return -ENODEV;
        }

        dst_addr = (struct sockaddr *)((char *)ev + sizeof(*ev));
        err = transport->send_ping(shost, ev->u.iscsi_ping.iface_num,
                                   ev->u.iscsi_ping.iface_type,
                                   ev->u.iscsi_ping.payload_size,
                                   ev->u.iscsi_ping.pid,
                                   dst_addr);
        scsi_host_put(shost);
        return err;
}

static int
iscsi_get_chap(struct iscsi_transport *transport, struct nlmsghdr *nlh)
{
        struct iscsi_uevent *ev = nlmsg_data(nlh);
        struct Scsi_Host *shost = NULL;
        struct iscsi_chap_rec *chap_rec;
        struct iscsi_internal *priv;
        struct sk_buff *skbchap;
        struct nlmsghdr *nlhchap;
        struct iscsi_uevent *evchap;
        uint32_t chap_buf_size;
        int len, err = 0;
        char *buf;

        if (!transport->get_chap)
                return -EINVAL;

        priv = iscsi_if_transport_lookup(transport);
        if (!priv)
                return -EINVAL;

        chap_buf_size = (ev->u.get_chap.num_entries * sizeof(*chap_rec));
        len = nlmsg_total_size(sizeof(*ev) + chap_buf_size);

        shost = scsi_host_lookup(ev->u.get_chap.host_no);
        if (!shost) {
                printk(KERN_ERR "%s: failed. Could not find host no %u\n",
                       __func__, ev->u.get_chap.host_no);
                return -ENODEV;
        }

        do {
                int actual_size;

                skbchap = alloc_skb(len, GFP_KERNEL);
                if (!skbchap) {
                        printk(KERN_ERR "can not deliver chap: OOM\n");
                        err = -ENOMEM;
                        goto exit_get_chap;
                }

                nlhchap = __nlmsg_put(skbchap, 0, 0, 0,
                                      (len - sizeof(*nlhchap)), 0);
                evchap = nlmsg_data(nlhchap);
                memset(evchap, 0, sizeof(*evchap));
                evchap->transport_handle = iscsi_handle(transport);
                evchap->type = nlh->nlmsg_type;
                evchap->u.get_chap.host_no = ev->u.get_chap.host_no;
                evchap->u.get_chap.chap_tbl_idx = ev->u.get_chap.chap_tbl_idx;
                evchap->u.get_chap.num_entries = ev->u.get_chap.num_entries;
                buf = (char *)evchap + sizeof(*evchap);
                memset(buf, 0, chap_buf_size);

                err = transport->get_chap(shost, ev->u.get_chap.chap_tbl_idx,
                                    &evchap->u.get_chap.num_entries, buf);

                actual_size = nlmsg_total_size(sizeof(*ev) + chap_buf_size);
                skb_trim(skbchap, NLMSG_ALIGN(actual_size));
                nlhchap->nlmsg_len = actual_size;

                err = iscsi_multicast_skb(skbchap, ISCSI_NL_GRP_ISCSID,
                                          GFP_KERNEL);
        } while (err < 0 && err != -ECONNREFUSED);

exit_get_chap:
        scsi_host_put(shost);
        return err;
}

static int iscsi_set_chap(struct iscsi_transport *transport,
                          struct iscsi_uevent *ev, uint32_t len)
{
        char *data = (char *)ev + sizeof(*ev);
        struct Scsi_Host *shost;
        int err = 0;

        if (!transport->set_chap)
                return -ENOSYS;

        shost = scsi_host_lookup(ev->u.set_path.host_no);
        if (!shost) {
                pr_err("%s could not find host no %u\n",
                       __func__, ev->u.set_path.host_no);
                return -ENODEV;
        }

        err = transport->set_chap(shost, data, len);
        scsi_host_put(shost);
        return err;
}

static int iscsi_delete_chap(struct iscsi_transport *transport,
                             struct iscsi_uevent *ev)
{
        struct Scsi_Host *shost;
        int err = 0;

        if (!transport->delete_chap)
                return -ENOSYS;

        shost = scsi_host_lookup(ev->u.delete_chap.host_no);
        if (!shost) {
                printk(KERN_ERR "%s could not find host no %u\n",
                       __func__, ev->u.delete_chap.host_no);
                return -ENODEV;
        }

        err = transport->delete_chap(shost, ev->u.delete_chap.chap_tbl_idx);
        scsi_host_put(shost);
        return err;
}

static const struct {
        enum iscsi_discovery_parent_type value;
        char                                *name;
} iscsi_discovery_parent_names[] = {
        {ISCSI_DISC_PARENT_UNKNOWN,        "Unknown" },
        {ISCSI_DISC_PARENT_SENDTGT,        "Sendtarget" },
        {ISCSI_DISC_PARENT_ISNS,        "isns" },
};

char *iscsi_get_discovery_parent_name(int parent_type)
{
        int i;
        char *state = "Unknown!";

        for (i = 0; i < ARRAY_SIZE(iscsi_discovery_parent_names); i++) {
                if (iscsi_discovery_parent_names[i].value & parent_type) {
                        state = iscsi_discovery_parent_names[i].name;
                        break;
                }
        }
        return state;
}
EXPORT_SYMBOL_GPL(iscsi_get_discovery_parent_name);

static int iscsi_set_flashnode_param(struct iscsi_transport *transport,
                                     struct iscsi_uevent *ev, uint32_t len)
{
        char *data = (char *)ev + sizeof(*ev);
        struct Scsi_Host *shost;
        struct iscsi_bus_flash_session *fnode_sess;
        struct iscsi_bus_flash_conn *fnode_conn;
        struct device *dev;
        uint32_t idx;
        int err = 0;

        if (!transport->set_flashnode_param) {
                err = -ENOSYS;
                goto exit_set_fnode;
        }

        shost = scsi_host_lookup(ev->u.set_flashnode.host_no);
        if (!shost) {
                pr_err("%s could not find host no %u\n",
                       __func__, ev->u.set_flashnode.host_no);
                err = -ENODEV;
                goto exit_set_fnode;
        }

        idx = ev->u.set_flashnode.flashnode_idx;
        fnode_sess = iscsi_get_flashnode_by_index(shost, idx);
        if (!fnode_sess) {
                pr_err("%s could not find flashnode %u for host no %u\n",
                       __func__, idx, ev->u.set_flashnode.host_no);
                err = -ENODEV;
                goto put_host;
        }

        dev = iscsi_find_flashnode_conn(fnode_sess);
        if (!dev) {
                err = -ENODEV;
                goto put_sess;
        }

        fnode_conn = iscsi_dev_to_flash_conn(dev);
        err = transport->set_flashnode_param(fnode_sess, fnode_conn, data, len);
        put_device(dev);

put_sess:
        put_device(&fnode_sess->dev);

put_host:
        scsi_host_put(shost);

exit_set_fnode:
        return err;
}

static int iscsi_new_flashnode(struct iscsi_transport *transport,
                               struct iscsi_uevent *ev, uint32_t len)
{
        char *data = (char *)ev + sizeof(*ev);
        struct Scsi_Host *shost;
        int index;
        int err = 0;

        if (!transport->new_flashnode) {
                err = -ENOSYS;
                goto exit_new_fnode;
        }

        shost = scsi_host_lookup(ev->u.new_flashnode.host_no);
        if (!shost) {
                pr_err("%s could not find host no %u\n",
                       __func__, ev->u.new_flashnode.host_no);
                err = -ENODEV;
                goto exit_new_fnode;
        }

        index = transport->new_flashnode(shost, data, len);

        if (index >= 0)
                ev->r.new_flashnode_ret.flashnode_idx = index;
        else
                err = -EIO;

        scsi_host_put(shost);

exit_new_fnode:
        return err;
}

static int iscsi_del_flashnode(struct iscsi_transport *transport,
                               struct iscsi_uevent *ev)
{
        struct Scsi_Host *shost;
        struct iscsi_bus_flash_session *fnode_sess;
        uint32_t idx;
        int err = 0;

        if (!transport->del_flashnode) {
                err = -ENOSYS;
                goto exit_del_fnode;
        }

        shost = scsi_host_lookup(ev->u.del_flashnode.host_no);
        if (!shost) {
                pr_err("%s could not find host no %u\n",
                       __func__, ev->u.del_flashnode.host_no);
                err = -ENODEV;
                goto exit_del_fnode;
        }

        idx = ev->u.del_flashnode.flashnode_idx;
        fnode_sess = iscsi_get_flashnode_by_index(shost, idx);
        if (!fnode_sess) {
                pr_err("%s could not find flashnode %u for host no %u\n",
                       __func__, idx, ev->u.del_flashnode.host_no);
                err = -ENODEV;
                goto put_host;
        }

        err = transport->del_flashnode(fnode_sess);
        put_device(&fnode_sess->dev);

put_host:
        scsi_host_put(shost);

exit_del_fnode:
        return err;
}

static int iscsi_login_flashnode(struct iscsi_transport *transport,
                                 struct iscsi_uevent *ev)
{
        struct Scsi_Host *shost;
        struct iscsi_bus_flash_session *fnode_sess;
        struct iscsi_bus_flash_conn *fnode_conn;
        struct device *dev;
        uint32_t idx;
        int err = 0;

        if (!transport->login_flashnode) {
                err = -ENOSYS;
                goto exit_login_fnode;
        }

        shost = scsi_host_lookup(ev->u.login_flashnode.host_no);
        if (!shost) {
                pr_err("%s could not find host no %u\n",
                       __func__, ev->u.login_flashnode.host_no);
                err = -ENODEV;
                goto exit_login_fnode;
        }

        idx = ev->u.login_flashnode.flashnode_idx;
        fnode_sess = iscsi_get_flashnode_by_index(shost, idx);
        if (!fnode_sess) {
                pr_err("%s could not find flashnode %u for host no %u\n",
                       __func__, idx, ev->u.login_flashnode.host_no);
                err = -ENODEV;
                goto put_host;
        }

        dev = iscsi_find_flashnode_conn(fnode_sess);
        if (!dev) {
                err = -ENODEV;
                goto put_sess;
        }

        fnode_conn = iscsi_dev_to_flash_conn(dev);
        err = transport->login_flashnode(fnode_sess, fnode_conn);
        put_device(dev);

put_sess:
        put_device(&fnode_sess->dev);

put_host:
        scsi_host_put(shost);

exit_login_fnode:
        return err;
}

static int iscsi_logout_flashnode(struct iscsi_transport *transport,
                                  struct iscsi_uevent *ev)
{
        struct Scsi_Host *shost;
        struct iscsi_bus_flash_session *fnode_sess;
        struct iscsi_bus_flash_conn *fnode_conn;
        struct device *dev;
        uint32_t idx;
        int err = 0;

        if (!transport->logout_flashnode) {
                err = -ENOSYS;
                goto exit_logout_fnode;
        }

        shost = scsi_host_lookup(ev->u.logout_flashnode.host_no);
        if (!shost) {
                pr_err("%s could not find host no %u\n",
                       __func__, ev->u.logout_flashnode.host_no);
                err = -ENODEV;
                goto exit_logout_fnode;
        }

        idx = ev->u.logout_flashnode.flashnode_idx;
        fnode_sess = iscsi_get_flashnode_by_index(shost, idx);
        if (!fnode_sess) {
                pr_err("%s could not find flashnode %u for host no %u\n",
                       __func__, idx, ev->u.logout_flashnode.host_no);
                err = -ENODEV;
                goto put_host;
        }

        dev = iscsi_find_flashnode_conn(fnode_sess);
        if (!dev) {
                err = -ENODEV;
                goto put_sess;
        }

        fnode_conn = iscsi_dev_to_flash_conn(dev);

        err = transport->logout_flashnode(fnode_sess, fnode_conn);
        put_device(dev);

put_sess:
        put_device(&fnode_sess->dev);

put_host:
        scsi_host_put(shost);

exit_logout_fnode:
        return err;
}

static int iscsi_logout_flashnode_sid(struct iscsi_transport *transport,
                                      struct iscsi_uevent *ev)
{
        struct Scsi_Host *shost;
        struct iscsi_cls_session *session;
        int err = 0;

        if (!transport->logout_flashnode_sid) {
                err = -ENOSYS;
                goto exit_logout_sid;
        }

        shost = scsi_host_lookup(ev->u.logout_flashnode_sid.host_no);
        if (!shost) {
                pr_err("%s could not find host no %u\n",
                       __func__, ev->u.logout_flashnode.host_no);
                err = -ENODEV;
                goto exit_logout_sid;
        }

        session = iscsi_session_lookup(ev->u.logout_flashnode_sid.sid);
        if (!session) {
                pr_err("%s could not find session id %u\n",
                       __func__, ev->u.logout_flashnode_sid.sid);
                err = -EINVAL;
                goto put_host;
        }

        err = transport->logout_flashnode_sid(session);

put_host:
        scsi_host_put(shost);

exit_logout_sid:
        return err;
}

static int
iscsi_get_host_stats(struct iscsi_transport *transport, struct nlmsghdr *nlh)
{
        struct iscsi_uevent *ev = nlmsg_data(nlh);
        struct Scsi_Host *shost = NULL;
        struct iscsi_internal *priv;
        struct sk_buff *skbhost_stats;
        struct nlmsghdr *nlhhost_stats;
        struct iscsi_uevent *evhost_stats;
        int host_stats_size = 0;
        int len, err = 0;
        char *buf;

        if (!transport->get_host_stats)
                return -ENOSYS;

        priv = iscsi_if_transport_lookup(transport);
        if (!priv)
                return -EINVAL;

        host_stats_size = sizeof(struct iscsi_offload_host_stats);
        len = nlmsg_total_size(sizeof(*ev) + host_stats_size);

        shost = scsi_host_lookup(ev->u.get_host_stats.host_no);
        if (!shost) {
                pr_err("%s: failed. Could not find host no %u\n",
                       __func__, ev->u.get_host_stats.host_no);
                return -ENODEV;
        }

        do {
                int actual_size;

                skbhost_stats = alloc_skb(len, GFP_KERNEL);
                if (!skbhost_stats) {
                        pr_err("cannot deliver host stats: OOM\n");
                        err = -ENOMEM;
                        goto exit_host_stats;
                }

                nlhhost_stats = __nlmsg_put(skbhost_stats, 0, 0, 0,
                                      (len - sizeof(*nlhhost_stats)), 0);
                evhost_stats = nlmsg_data(nlhhost_stats);
                memset(evhost_stats, 0, sizeof(*evhost_stats));
                evhost_stats->transport_handle = iscsi_handle(transport);
                evhost_stats->type = nlh->nlmsg_type;
                evhost_stats->u.get_host_stats.host_no =
                                        ev->u.get_host_stats.host_no;
                buf = (char *)evhost_stats + sizeof(*evhost_stats);
                memset(buf, 0, host_stats_size);

                err = transport->get_host_stats(shost, buf, host_stats_size);
                if (err) {
                        kfree_skb(skbhost_stats);
                        goto exit_host_stats;
                }

                actual_size = nlmsg_total_size(sizeof(*ev) + host_stats_size);
                skb_trim(skbhost_stats, NLMSG_ALIGN(actual_size));
                nlhhost_stats->nlmsg_len = actual_size;

                err = iscsi_multicast_skb(skbhost_stats, ISCSI_NL_GRP_ISCSID,
                                          GFP_KERNEL);
        } while (err < 0 && err != -ECONNREFUSED);

exit_host_stats:
        scsi_host_put(shost);
        return err;
}

static int iscsi_if_transport_conn(struct iscsi_transport *transport,
                                   struct nlmsghdr *nlh, u32 pdu_len)
{
        struct iscsi_uevent *ev = nlmsg_data(nlh);
        struct iscsi_cls_session *session;
        struct iscsi_cls_conn *conn = NULL;
        struct iscsi_endpoint *ep;
        int err = 0;

        switch (nlh->nlmsg_type) {
        case ISCSI_UEVENT_CREATE_CONN:
                return iscsi_if_create_conn(transport, ev);
        case ISCSI_UEVENT_DESTROY_CONN:
                return iscsi_if_destroy_conn(transport, ev);
        case ISCSI_UEVENT_STOP_CONN:
                conn = iscsi_conn_lookup(ev->u.stop_conn.sid,
                                         ev->u.stop_conn.cid);
                if (!conn)
                        return -EINVAL;

                return iscsi_if_stop_conn(conn, ev->u.stop_conn.flag);
        }

        /*
         * The following cmds need to be run under the ep_mutex so in kernel
         * conn cleanup (ep_disconnect + unbind and conn) is not done while
         * these are running. They also must not run if we have just run a conn
         * cleanup because they would set the state in a way that might allow
         * IO or send IO themselves.
         */
        switch (nlh->nlmsg_type) {
        case ISCSI_UEVENT_START_CONN:
                conn = iscsi_conn_lookup(ev->u.start_conn.sid,
                                         ev->u.start_conn.cid);
                break;
        case ISCSI_UEVENT_BIND_CONN:
                conn = iscsi_conn_lookup(ev->u.b_conn.sid, ev->u.b_conn.cid);
                break;
        case ISCSI_UEVENT_SEND_PDU:
                conn = iscsi_conn_lookup(ev->u.send_pdu.sid, ev->u.send_pdu.cid);
                break;
        }

        if (!conn)
                return -EINVAL;

        mutex_lock(&conn->ep_mutex);
        spin_lock_irq(&conn->lock);
        if (test_bit(ISCSI_CLS_CONN_BIT_CLEANUP, &conn->flags)) {
                spin_unlock_irq(&conn->lock);
                mutex_unlock(&conn->ep_mutex);
                ev->r.retcode = -ENOTCONN;
                return 0;
        }
        spin_unlock_irq(&conn->lock);

        switch (nlh->nlmsg_type) {
        case ISCSI_UEVENT_BIND_CONN:
                session = iscsi_session_lookup(ev->u.b_conn.sid);
                if (!session) {
                        err = -EINVAL;
                        break;
                }

                ev->r.retcode =        transport->bind_conn(session, conn,
                                                ev->u.b_conn.transport_eph,
                                                ev->u.b_conn.is_leading);
                if (!ev->r.retcode)
                        WRITE_ONCE(conn->state, ISCSI_CONN_BOUND);

                if (ev->r.retcode || !transport->ep_connect)
                        break;

                ep = iscsi_lookup_endpoint(ev->u.b_conn.transport_eph);
                if (ep) {
                        ep->conn = conn;
                        conn->ep = ep;
                        iscsi_put_endpoint(ep);
                } else {
                        err = -ENOTCONN;
                        iscsi_cls_conn_printk(KERN_ERR, conn,
                                              "Could not set ep conn binding\n");
                }
                break;
        case ISCSI_UEVENT_START_CONN:
                ev->r.retcode = transport->start_conn(conn);
                if (!ev->r.retcode)
                        WRITE_ONCE(conn->state, ISCSI_CONN_UP);

                break;
        case ISCSI_UEVENT_SEND_PDU:
                if ((ev->u.send_pdu.hdr_size > pdu_len) ||
                    (ev->u.send_pdu.data_size > (pdu_len - ev->u.send_pdu.hdr_size))) {
                        err = -EINVAL;
                        break;
                }

                ev->r.retcode =        transport->send_pdu(conn,
                                (struct iscsi_hdr *)((char *)ev + sizeof(*ev)),
                                (char *)ev + sizeof(*ev) + ev->u.send_pdu.hdr_size,
                                ev->u.send_pdu.data_size);
                break;
        default:
                err = -ENOSYS;
        }

        mutex_unlock(&conn->ep_mutex);
        return err;
}

static int
iscsi_if_recv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, uint32_t *group)
{
        int err = 0;
        u32 portid;
        struct iscsi_uevent *ev = nlmsg_data(nlh);
        struct iscsi_transport *transport = NULL;
        struct iscsi_internal *priv;
        struct iscsi_cls_session *session;
        struct iscsi_endpoint *ep = NULL;
        u32 rlen;

        if (!netlink_capable(skb, CAP_SYS_ADMIN))
                return -EPERM;

        if (nlh->nlmsg_type == ISCSI_UEVENT_PATH_UPDATE)
                *group = ISCSI_NL_GRP_UIP;
        else
                *group = ISCSI_NL_GRP_ISCSID;

        priv = iscsi_if_transport_lookup(iscsi_ptr(ev->transport_handle));
        if (!priv)
                return -EINVAL;
        transport = priv->iscsi_transport;

        if (!try_module_get(transport->owner))
                return -EINVAL;

        portid = NETLINK_CB(skb).portid;

        /*
         * Even though the remaining payload may not be regarded as nlattr,
         * (like address or something else), calculate the remaining length
         * here to ease following length checks.
         */
        rlen = nlmsg_attrlen(nlh, sizeof(*ev));

        switch (nlh->nlmsg_type) {
        case ISCSI_UEVENT_CREATE_SESSION:
                err = iscsi_if_create_session(priv, ep, ev,
                                              portid,
                                              ev->u.c_session.initial_cmdsn,
                                              ev->u.c_session.cmds_max,
                                              ev->u.c_session.queue_depth);
                break;
        case ISCSI_UEVENT_CREATE_BOUND_SESSION:
                ep = iscsi_lookup_endpoint(ev->u.c_bound_session.ep_handle);
                if (!ep) {
                        err = -EINVAL;
                        break;
                }

                err = iscsi_if_create_session(priv, ep, ev,
                                        portid,
                                        ev->u.c_bound_session.initial_cmdsn,
                                        ev->u.c_bound_session.cmds_max,
                                        ev->u.c_bound_session.queue_depth);
                iscsi_put_endpoint(ep);
                break;
        case ISCSI_UEVENT_DESTROY_SESSION:
                session = iscsi_session_lookup(ev->u.d_session.sid);
                if (!session)
                        err = -EINVAL;
                else if (iscsi_session_has_conns(ev->u.d_session.sid))
                        err = -EBUSY;
                else
                        transport->destroy_session(session);
                break;
        case ISCSI_UEVENT_DESTROY_SESSION_ASYNC:
                session = iscsi_session_lookup(ev->u.d_session.sid);
                if (!session)
                        err = -EINVAL;
                else if (iscsi_session_has_conns(ev->u.d_session.sid))
                        err = -EBUSY;
                else {
                        unsigned long flags;

                        /* Prevent this session from being found again */
                        spin_lock_irqsave(&sesslock, flags);
                        list_del_init(&session->sess_list);
                        spin_unlock_irqrestore(&sesslock, flags);

                        queue_work(system_dfl_wq, &session->destroy_work);
                }
                break;
        case ISCSI_UEVENT_UNBIND_SESSION:
                session = iscsi_session_lookup(ev->u.d_session.sid);
                if (session)
                        queue_work(session->workq, &session->unbind_work);
                else
                        err = -EINVAL;
                break;
        case ISCSI_UEVENT_SET_PARAM:
                err = iscsi_if_set_param(transport, ev, rlen);
                break;
        case ISCSI_UEVENT_CREATE_CONN:
        case ISCSI_UEVENT_DESTROY_CONN:
        case ISCSI_UEVENT_STOP_CONN:
        case ISCSI_UEVENT_START_CONN:
        case ISCSI_UEVENT_BIND_CONN:
        case ISCSI_UEVENT_SEND_PDU:
                err = iscsi_if_transport_conn(transport, nlh, rlen);
                break;
        case ISCSI_UEVENT_GET_STATS:
                err = iscsi_if_get_stats(transport, nlh);
                break;
        case ISCSI_UEVENT_TRANSPORT_EP_CONNECT:
        case ISCSI_UEVENT_TRANSPORT_EP_POLL:
        case ISCSI_UEVENT_TRANSPORT_EP_DISCONNECT:
        case ISCSI_UEVENT_TRANSPORT_EP_CONNECT_THROUGH_HOST:
                err = iscsi_if_transport_ep(transport, ev, nlh->nlmsg_type, rlen);
                break;
        case ISCSI_UEVENT_TGT_DSCVR:
                err = iscsi_tgt_dscvr(transport, ev, rlen);
                break;
        case ISCSI_UEVENT_SET_HOST_PARAM:
                err = iscsi_set_host_param(transport, ev, rlen);
                break;
        case ISCSI_UEVENT_PATH_UPDATE:
                err = iscsi_set_path(transport, ev, rlen);
                break;
        case ISCSI_UEVENT_SET_IFACE_PARAMS:
                err = iscsi_set_iface_params(transport, ev, rlen);
                break;
        case ISCSI_UEVENT_PING:
                err = iscsi_send_ping(transport, ev, rlen);
                break;
        case ISCSI_UEVENT_GET_CHAP:
                err = iscsi_get_chap(transport, nlh);
                break;
        case ISCSI_UEVENT_DELETE_CHAP:
                err = iscsi_delete_chap(transport, ev);
                break;
        case ISCSI_UEVENT_SET_FLASHNODE_PARAMS:
                err = iscsi_set_flashnode_param(transport, ev, rlen);
                break;
        case ISCSI_UEVENT_NEW_FLASHNODE:
                err = iscsi_new_flashnode(transport, ev, rlen);
                break;
        case ISCSI_UEVENT_DEL_FLASHNODE:
                err = iscsi_del_flashnode(transport, ev);
                break;
        case ISCSI_UEVENT_LOGIN_FLASHNODE:
                err = iscsi_login_flashnode(transport, ev);
                break;
        case ISCSI_UEVENT_LOGOUT_FLASHNODE:
                err = iscsi_logout_flashnode(transport, ev);
                break;
        case ISCSI_UEVENT_LOGOUT_FLASHNODE_SID:
                err = iscsi_logout_flashnode_sid(transport, ev);
                break;
        case ISCSI_UEVENT_SET_CHAP:
                err = iscsi_set_chap(transport, ev, rlen);
                break;
        case ISCSI_UEVENT_GET_HOST_STATS:
                err = iscsi_get_host_stats(transport, nlh);
                break;
        default:
                err = -ENOSYS;
                break;
        }

        module_put(transport->owner);
        return err;
}

/*
 * Get message from skb.  Each message is processed by iscsi_if_recv_msg.
 * Malformed skbs with wrong lengths or invalid creds are not processed.
 */
static void
iscsi_if_rx(struct sk_buff *skb)
{
        u32 portid = NETLINK_CB(skb).portid;

        mutex_lock(&rx_queue_mutex);
        while (skb->len >= NLMSG_HDRLEN) {
                int err;
                uint32_t rlen;
                struct nlmsghdr        *nlh;
                struct iscsi_uevent *ev;
                uint32_t group;
                int retries = ISCSI_SEND_MAX_ALLOWED;

                nlh = nlmsg_hdr(skb);
                if (nlh->nlmsg_len < sizeof(*nlh) + sizeof(*ev) ||
                    skb->len < nlh->nlmsg_len) {
                        break;
                }

                ev = nlmsg_data(nlh);
                rlen = NLMSG_ALIGN(nlh->nlmsg_len);
                if (rlen > skb->len)
                        rlen = skb->len;

                err = iscsi_if_recv_msg(skb, nlh, &group);
                if (err) {
                        ev->type = ISCSI_KEVENT_IF_ERROR;
                        ev->iferror = err;
                }
                do {
                        /*
                         * special case for GET_STATS, GET_CHAP and GET_HOST_STATS:
                         * on success - sending reply and stats from
                         * inside of if_recv_msg(),
                         * on error - fall through.
                         */
                        if (ev->type == ISCSI_UEVENT_GET_STATS && !err)
                                break;
                        if (ev->type == ISCSI_UEVENT_GET_CHAP && !err)
                                break;
                        if (ev->type == ISCSI_UEVENT_GET_HOST_STATS && !err)
                                break;
                        err = iscsi_if_send_reply(portid, nlh->nlmsg_type,
                                                  ev, sizeof(*ev));
                        if (err == -EAGAIN && --retries < 0) {
                                printk(KERN_WARNING "Send reply failed, error %d\n", err);
                                break;
                        }
                } while (err < 0 && err != -ECONNREFUSED && err != -ESRCH);
                skb_pull(skb, rlen);
        }
        mutex_unlock(&rx_queue_mutex);
}

#define ISCSI_CLASS_ATTR(_prefix,_name,_mode,_show,_store)                \
struct device_attribute dev_attr_##_prefix##_##_name =        \
        __ATTR(_name,_mode,_show,_store)

/*
 * iSCSI connection attrs
 */
#define iscsi_conn_attr_show(param)                                        \
static ssize_t                                                                \
show_conn_param_##param(struct device *dev,                                 \
                        struct device_attribute *attr, char *buf)        \
{                                                                        \
        struct iscsi_cls_conn *conn = iscsi_dev_to_conn(dev->parent);        \
        struct iscsi_transport *t = conn->transport;                        \
        return t->get_conn_param(conn, param, buf);                        \
}

#define iscsi_conn_attr(field, param)                                        \
        iscsi_conn_attr_show(param)                                        \
static ISCSI_CLASS_ATTR(conn, field, S_IRUGO, show_conn_param_##param,        \
                        NULL);

iscsi_conn_attr(max_recv_dlength, ISCSI_PARAM_MAX_RECV_DLENGTH);
iscsi_conn_attr(max_xmit_dlength, ISCSI_PARAM_MAX_XMIT_DLENGTH);
iscsi_conn_attr(header_digest, ISCSI_PARAM_HDRDGST_EN);
iscsi_conn_attr(data_digest, ISCSI_PARAM_DATADGST_EN);
iscsi_conn_attr(ifmarker, ISCSI_PARAM_IFMARKER_EN);
iscsi_conn_attr(ofmarker, ISCSI_PARAM_OFMARKER_EN);
iscsi_conn_attr(persistent_port, ISCSI_PARAM_PERSISTENT_PORT);
iscsi_conn_attr(exp_statsn, ISCSI_PARAM_EXP_STATSN);
iscsi_conn_attr(persistent_address, ISCSI_PARAM_PERSISTENT_ADDRESS);
iscsi_conn_attr(ping_tmo, ISCSI_PARAM_PING_TMO);
iscsi_conn_attr(recv_tmo, ISCSI_PARAM_RECV_TMO);
iscsi_conn_attr(local_port, ISCSI_PARAM_LOCAL_PORT);
iscsi_conn_attr(statsn, ISCSI_PARAM_STATSN);
iscsi_conn_attr(keepalive_tmo, ISCSI_PARAM_KEEPALIVE_TMO);
iscsi_conn_attr(max_segment_size, ISCSI_PARAM_MAX_SEGMENT_SIZE);
iscsi_conn_attr(tcp_timestamp_stat, ISCSI_PARAM_TCP_TIMESTAMP_STAT);
iscsi_conn_attr(tcp_wsf_disable, ISCSI_PARAM_TCP_WSF_DISABLE);
iscsi_conn_attr(tcp_nagle_disable, ISCSI_PARAM_TCP_NAGLE_DISABLE);
iscsi_conn_attr(tcp_timer_scale, ISCSI_PARAM_TCP_TIMER_SCALE);
iscsi_conn_attr(tcp_timestamp_enable, ISCSI_PARAM_TCP_TIMESTAMP_EN);
iscsi_conn_attr(fragment_disable, ISCSI_PARAM_IP_FRAGMENT_DISABLE);
iscsi_conn_attr(ipv4_tos, ISCSI_PARAM_IPV4_TOS);
iscsi_conn_attr(ipv6_traffic_class, ISCSI_PARAM_IPV6_TC);
iscsi_conn_attr(ipv6_flow_label, ISCSI_PARAM_IPV6_FLOW_LABEL);
iscsi_conn_attr(is_fw_assigned_ipv6, ISCSI_PARAM_IS_FW_ASSIGNED_IPV6);
iscsi_conn_attr(tcp_xmit_wsf, ISCSI_PARAM_TCP_XMIT_WSF);
iscsi_conn_attr(tcp_recv_wsf, ISCSI_PARAM_TCP_RECV_WSF);
iscsi_conn_attr(local_ipaddr, ISCSI_PARAM_LOCAL_IPADDR);

static const char *const connection_state_names[] = {
        [ISCSI_CONN_UP] = "up",
        [ISCSI_CONN_DOWN] = "down",
        [ISCSI_CONN_FAILED] = "failed",
        [ISCSI_CONN_BOUND] = "bound"
};

static ssize_t show_conn_state(struct device *dev,
                               struct device_attribute *attr, char *buf)
{
        struct iscsi_cls_conn *conn = iscsi_dev_to_conn(dev->parent);
        const char *state = "unknown";
        int conn_state = READ_ONCE(conn->state);

        if (conn_state >= 0 &&
            conn_state < ARRAY_SIZE(connection_state_names))
                state = connection_state_names[conn_state];

        return sysfs_emit(buf, "%s\n", state);
}
static ISCSI_CLASS_ATTR(conn, state, S_IRUGO, show_conn_state,
                        NULL);

#define iscsi_conn_ep_attr_show(param)                                        \
static ssize_t show_conn_ep_param_##param(struct device *dev,                \
                                          struct device_attribute *attr,\
                                          char *buf)                        \
{                                                                        \
        struct iscsi_cls_conn *conn = iscsi_dev_to_conn(dev->parent);        \
        struct iscsi_transport *t = conn->transport;                        \
        struct iscsi_endpoint *ep;                                        \
        ssize_t rc;                                                        \
                                                                        \
        /*                                                                \
         * Need to make sure ep_disconnect does not free the LLD's        \
         * interconnect resources while we are trying to read them.        \
         */                                                                \
        mutex_lock(&conn->ep_mutex);                                        \
        ep = conn->ep;                                                        \
        if (!ep && t->ep_connect) {                                        \
                mutex_unlock(&conn->ep_mutex);                                \
                return -ENOTCONN;                                        \
        }                                                                \
                                                                        \
        if (ep)                                                                \
                rc = t->get_ep_param(ep, param, buf);                        \
        else                                                                \
                rc = t->get_conn_param(conn, param, buf);                \
        mutex_unlock(&conn->ep_mutex);                                        \
        return rc;                                                        \
}

#define iscsi_conn_ep_attr(field, param)                                \
        iscsi_conn_ep_attr_show(param)                                        \
static ISCSI_CLASS_ATTR(conn, field, S_IRUGO,                                \
                        show_conn_ep_param_##param, NULL);

iscsi_conn_ep_attr(address, ISCSI_PARAM_CONN_ADDRESS);
iscsi_conn_ep_attr(port, ISCSI_PARAM_CONN_PORT);

static struct attribute *iscsi_conn_attrs[] = {
        &dev_attr_conn_max_recv_dlength.attr,
        &dev_attr_conn_max_xmit_dlength.attr,
        &dev_attr_conn_header_digest.attr,
        &dev_attr_conn_data_digest.attr,
        &dev_attr_conn_ifmarker.attr,
        &dev_attr_conn_ofmarker.attr,
        &dev_attr_conn_address.attr,
        &dev_attr_conn_port.attr,
        &dev_attr_conn_exp_statsn.attr,
        &dev_attr_conn_persistent_address.attr,
        &dev_attr_conn_persistent_port.attr,
        &dev_attr_conn_ping_tmo.attr,
        &dev_attr_conn_recv_tmo.attr,
        &dev_attr_conn_local_port.attr,
        &dev_attr_conn_statsn.attr,
        &dev_attr_conn_keepalive_tmo.attr,
        &dev_attr_conn_max_segment_size.attr,
        &dev_attr_conn_tcp_timestamp_stat.attr,
        &dev_attr_conn_tcp_wsf_disable.attr,
        &dev_attr_conn_tcp_nagle_disable.attr,
        &dev_attr_conn_tcp_timer_scale.attr,
        &dev_attr_conn_tcp_timestamp_enable.attr,
        &dev_attr_conn_fragment_disable.attr,
        &dev_attr_conn_ipv4_tos.attr,
        &dev_attr_conn_ipv6_traffic_class.attr,
        &dev_attr_conn_ipv6_flow_label.attr,
        &dev_attr_conn_is_fw_assigned_ipv6.attr,
        &dev_attr_conn_tcp_xmit_wsf.attr,
        &dev_attr_conn_tcp_recv_wsf.attr,
        &dev_attr_conn_local_ipaddr.attr,
        &dev_attr_conn_state.attr,
        NULL,
};

static umode_t iscsi_conn_attr_is_visible(struct kobject *kobj,
                                         struct attribute *attr, int i)
{
        struct device *cdev = container_of(kobj, struct device, kobj);
        struct iscsi_cls_conn *conn = transport_class_to_conn(cdev);
        struct iscsi_transport *t = conn->transport;
        int param;

        if (attr == &dev_attr_conn_max_recv_dlength.attr)
                param = ISCSI_PARAM_MAX_RECV_DLENGTH;
        else if (attr == &dev_attr_conn_max_xmit_dlength.attr)
                param = ISCSI_PARAM_MAX_XMIT_DLENGTH;
        else if (attr == &dev_attr_conn_header_digest.attr)
                param = ISCSI_PARAM_HDRDGST_EN;
        else if (attr == &dev_attr_conn_data_digest.attr)
                param = ISCSI_PARAM_DATADGST_EN;
        else if (attr == &dev_attr_conn_ifmarker.attr)
                param = ISCSI_PARAM_IFMARKER_EN;
        else if (attr == &dev_attr_conn_ofmarker.attr)
                param = ISCSI_PARAM_OFMARKER_EN;
        else if (attr == &dev_attr_conn_address.attr)
                param = ISCSI_PARAM_CONN_ADDRESS;
        else if (attr == &dev_attr_conn_port.attr)
                param = ISCSI_PARAM_CONN_PORT;
        else if (attr == &dev_attr_conn_exp_statsn.attr)
                param = ISCSI_PARAM_EXP_STATSN;
        else if (attr == &dev_attr_conn_persistent_address.attr)
                param = ISCSI_PARAM_PERSISTENT_ADDRESS;
        else if (attr == &dev_attr_conn_persistent_port.attr)
                param = ISCSI_PARAM_PERSISTENT_PORT;
        else if (attr == &dev_attr_conn_ping_tmo.attr)
                param = ISCSI_PARAM_PING_TMO;
        else if (attr == &dev_attr_conn_recv_tmo.attr)
                param = ISCSI_PARAM_RECV_TMO;
        else if (attr == &dev_attr_conn_local_port.attr)
                param = ISCSI_PARAM_LOCAL_PORT;
        else if (attr == &dev_attr_conn_statsn.attr)
                param = ISCSI_PARAM_STATSN;
        else if (attr == &dev_attr_conn_keepalive_tmo.attr)
                param = ISCSI_PARAM_KEEPALIVE_TMO;
        else if (attr == &dev_attr_conn_max_segment_size.attr)
                param = ISCSI_PARAM_MAX_SEGMENT_SIZE;
        else if (attr == &dev_attr_conn_tcp_timestamp_stat.attr)
                param = ISCSI_PARAM_TCP_TIMESTAMP_STAT;
        else if (attr == &dev_attr_conn_tcp_wsf_disable.attr)
                param = ISCSI_PARAM_TCP_WSF_DISABLE;
        else if (attr == &dev_attr_conn_tcp_nagle_disable.attr)
                param = ISCSI_PARAM_TCP_NAGLE_DISABLE;
        else if (attr == &dev_attr_conn_tcp_timer_scale.attr)
                param = ISCSI_PARAM_TCP_TIMER_SCALE;
        else if (attr == &dev_attr_conn_tcp_timestamp_enable.attr)
                param = ISCSI_PARAM_TCP_TIMESTAMP_EN;
        else if (attr == &dev_attr_conn_fragment_disable.attr)
                param = ISCSI_PARAM_IP_FRAGMENT_DISABLE;
        else if (attr == &dev_attr_conn_ipv4_tos.attr)
                param = ISCSI_PARAM_IPV4_TOS;
        else if (attr == &dev_attr_conn_ipv6_traffic_class.attr)
                param = ISCSI_PARAM_IPV6_TC;
        else if (attr == &dev_attr_conn_ipv6_flow_label.attr)
                param = ISCSI_PARAM_IPV6_FLOW_LABEL;
        else if (attr == &dev_attr_conn_is_fw_assigned_ipv6.attr)
                param = ISCSI_PARAM_IS_FW_ASSIGNED_IPV6;
        else if (attr == &dev_attr_conn_tcp_xmit_wsf.attr)
                param = ISCSI_PARAM_TCP_XMIT_WSF;
        else if (attr == &dev_attr_conn_tcp_recv_wsf.attr)
                param = ISCSI_PARAM_TCP_RECV_WSF;
        else if (attr == &dev_attr_conn_local_ipaddr.attr)
                param = ISCSI_PARAM_LOCAL_IPADDR;
        else if (attr == &dev_attr_conn_state.attr)
                return S_IRUGO;
        else {
                WARN_ONCE(1, "Invalid conn attr");
                return 0;
        }

        return t->attr_is_visible(ISCSI_PARAM, param);
}

static struct attribute_group iscsi_conn_group = {
        .attrs = iscsi_conn_attrs,
        .is_visible = iscsi_conn_attr_is_visible,
};

/*
 * iSCSI session attrs
 */
#define iscsi_session_attr_show(param, perm)                                \
static ssize_t                                                                \
show_session_param_##param(struct device *dev,                                \
                           struct device_attribute *attr, char *buf)        \
{                                                                        \
        struct iscsi_cls_session *session =                                 \
                iscsi_dev_to_session(dev->parent);                        \
        struct iscsi_transport *t = session->transport;                        \
                                                                        \
        if (perm && !capable(CAP_SYS_ADMIN))                                \
                return -EACCES;                                                \
        return t->get_session_param(session, param, buf);                \
}

#define iscsi_session_attr(field, param, perm)                                \
        iscsi_session_attr_show(param, perm)                                \
static ISCSI_CLASS_ATTR(sess, field, S_IRUGO, show_session_param_##param, \
                        NULL);
iscsi_session_attr(targetname, ISCSI_PARAM_TARGET_NAME, 0);
iscsi_session_attr(initial_r2t, ISCSI_PARAM_INITIAL_R2T_EN, 0);
iscsi_session_attr(max_outstanding_r2t, ISCSI_PARAM_MAX_R2T, 0);
iscsi_session_attr(immediate_data, ISCSI_PARAM_IMM_DATA_EN, 0);
iscsi_session_attr(first_burst_len, ISCSI_PARAM_FIRST_BURST, 0);
iscsi_session_attr(max_burst_len, ISCSI_PARAM_MAX_BURST, 0);
iscsi_session_attr(data_pdu_in_order, ISCSI_PARAM_PDU_INORDER_EN, 0);
iscsi_session_attr(data_seq_in_order, ISCSI_PARAM_DATASEQ_INORDER_EN, 0);
iscsi_session_attr(erl, ISCSI_PARAM_ERL, 0);
iscsi_session_attr(tpgt, ISCSI_PARAM_TPGT, 0);
iscsi_session_attr(username, ISCSI_PARAM_USERNAME, 1);
iscsi_session_attr(username_in, ISCSI_PARAM_USERNAME_IN, 1);
iscsi_session_attr(password, ISCSI_PARAM_PASSWORD, 1);
iscsi_session_attr(password_in, ISCSI_PARAM_PASSWORD_IN, 1);
iscsi_session_attr(chap_out_idx, ISCSI_PARAM_CHAP_OUT_IDX, 1);
iscsi_session_attr(chap_in_idx, ISCSI_PARAM_CHAP_IN_IDX, 1);
iscsi_session_attr(fast_abort, ISCSI_PARAM_FAST_ABORT, 0);
iscsi_session_attr(abort_tmo, ISCSI_PARAM_ABORT_TMO, 0);
iscsi_session_attr(lu_reset_tmo, ISCSI_PARAM_LU_RESET_TMO, 0);
iscsi_session_attr(tgt_reset_tmo, ISCSI_PARAM_TGT_RESET_TMO, 0);
iscsi_session_attr(ifacename, ISCSI_PARAM_IFACE_NAME, 0);
iscsi_session_attr(initiatorname, ISCSI_PARAM_INITIATOR_NAME, 0);
iscsi_session_attr(targetalias, ISCSI_PARAM_TARGET_ALIAS, 0);
iscsi_session_attr(boot_root, ISCSI_PARAM_BOOT_ROOT, 0);
iscsi_session_attr(boot_nic, ISCSI_PARAM_BOOT_NIC, 0);
iscsi_session_attr(boot_target, ISCSI_PARAM_BOOT_TARGET, 0);
iscsi_session_attr(auto_snd_tgt_disable, ISCSI_PARAM_AUTO_SND_TGT_DISABLE, 0);
iscsi_session_attr(discovery_session, ISCSI_PARAM_DISCOVERY_SESS, 0);
iscsi_session_attr(portal_type, ISCSI_PARAM_PORTAL_TYPE, 0);
iscsi_session_attr(chap_auth, ISCSI_PARAM_CHAP_AUTH_EN, 0);
iscsi_session_attr(discovery_logout, ISCSI_PARAM_DISCOVERY_LOGOUT_EN, 0);
iscsi_session_attr(bidi_chap, ISCSI_PARAM_BIDI_CHAP_EN, 0);
iscsi_session_attr(discovery_auth_optional,
                   ISCSI_PARAM_DISCOVERY_AUTH_OPTIONAL, 0);
iscsi_session_attr(def_time2wait, ISCSI_PARAM_DEF_TIME2WAIT, 0);
iscsi_session_attr(def_time2retain, ISCSI_PARAM_DEF_TIME2RETAIN, 0);
iscsi_session_attr(isid, ISCSI_PARAM_ISID, 0);
iscsi_session_attr(tsid, ISCSI_PARAM_TSID, 0);
iscsi_session_attr(def_taskmgmt_tmo, ISCSI_PARAM_DEF_TASKMGMT_TMO, 0);
iscsi_session_attr(discovery_parent_idx, ISCSI_PARAM_DISCOVERY_PARENT_IDX, 0);
iscsi_session_attr(discovery_parent_type, ISCSI_PARAM_DISCOVERY_PARENT_TYPE, 0);

static ssize_t
show_priv_session_target_state(struct device *dev, struct device_attribute *attr,
                        char *buf)
{
        struct iscsi_cls_session *session = iscsi_dev_to_session(dev->parent);

        return sysfs_emit(buf, "%s\n",
                        iscsi_session_target_state_name[session->target_state]);
}

static ISCSI_CLASS_ATTR(priv_sess, target_state, S_IRUGO,
                        show_priv_session_target_state, NULL);

static ssize_t
show_priv_session_state(struct device *dev, struct device_attribute *attr,
                        char *buf)
{
        struct iscsi_cls_session *session = iscsi_dev_to_session(dev->parent);
        return sysfs_emit(buf, "%s\n", iscsi_session_state_name(session->state));
}
static ISCSI_CLASS_ATTR(priv_sess, state, S_IRUGO, show_priv_session_state,
                        NULL);
static ssize_t
show_priv_session_creator(struct device *dev, struct device_attribute *attr,
                        char *buf)
{
        struct iscsi_cls_session *session = iscsi_dev_to_session(dev->parent);
        return sysfs_emit(buf, "%d\n", session->creator);
}
static ISCSI_CLASS_ATTR(priv_sess, creator, S_IRUGO, show_priv_session_creator,
                        NULL);
static ssize_t
show_priv_session_target_id(struct device *dev, struct device_attribute *attr,
                            char *buf)
{
        struct iscsi_cls_session *session = iscsi_dev_to_session(dev->parent);
        return sysfs_emit(buf, "%d\n", session->target_id);
}
static ISCSI_CLASS_ATTR(priv_sess, target_id, S_IRUGO,
                        show_priv_session_target_id, NULL);

#define iscsi_priv_session_attr_show(field, format)                        \
static ssize_t                                                                \
show_priv_session_##field(struct device *dev,                                 \
                          struct device_attribute *attr, char *buf)        \
{                                                                        \
        struct iscsi_cls_session *session =                                 \
                        iscsi_dev_to_session(dev->parent);                \
        if (session->field == -1)                                        \
                return sysfs_emit(buf, "off\n");                        \
        return sysfs_emit(buf, format"\n", session->field);                \
}

#define iscsi_priv_session_attr_store(field)                                \
static ssize_t                                                                \
store_priv_session_##field(struct device *dev,                                \
                           struct device_attribute *attr,                \
                           const char *buf, size_t count)                \
{                                                                        \
        int val;                                                        \
        char *cp;                                                        \
        struct iscsi_cls_session *session =                                \
                iscsi_dev_to_session(dev->parent);                        \
        if ((session->state == ISCSI_SESSION_FREE) ||                        \
            (session->state == ISCSI_SESSION_FAILED))                        \
                return -EBUSY;                                                \
        if (strncmp(buf, "off", 3) == 0) {                                \
                session->field = -1;                                        \
                session->field##_sysfs_override = true;                        \
        } else {                                                        \
                val = simple_strtoul(buf, &cp, 0);                        \
                if (*cp != '\0' && *cp != '\n')                                \
                        return -EINVAL;                                        \
                session->field = val;                                        \
                session->field##_sysfs_override = true;                        \
        }                                                                \
        return count;                                                        \
}

#define iscsi_priv_session_rw_attr(field, format)                        \
        iscsi_priv_session_attr_show(field, format)                        \
        iscsi_priv_session_attr_store(field)                                \
static ISCSI_CLASS_ATTR(priv_sess, field, S_IRUGO | S_IWUSR,                \
                        show_priv_session_##field,                        \
                        store_priv_session_##field)

iscsi_priv_session_rw_attr(recovery_tmo, "%d");

static struct attribute *iscsi_session_attrs[] = {
        &dev_attr_sess_initial_r2t.attr,
        &dev_attr_sess_max_outstanding_r2t.attr,
        &dev_attr_sess_immediate_data.attr,
        &dev_attr_sess_first_burst_len.attr,
        &dev_attr_sess_max_burst_len.attr,
        &dev_attr_sess_data_pdu_in_order.attr,
        &dev_attr_sess_data_seq_in_order.attr,
        &dev_attr_sess_erl.attr,
        &dev_attr_sess_targetname.attr,
        &dev_attr_sess_tpgt.attr,
        &dev_attr_sess_password.attr,
        &dev_attr_sess_password_in.attr,
        &dev_attr_sess_username.attr,
        &dev_attr_sess_username_in.attr,
        &dev_attr_sess_fast_abort.attr,
        &dev_attr_sess_abort_tmo.attr,
        &dev_attr_sess_lu_reset_tmo.attr,
        &dev_attr_sess_tgt_reset_tmo.attr,
        &dev_attr_sess_ifacename.attr,
        &dev_attr_sess_initiatorname.attr,
        &dev_attr_sess_targetalias.attr,
        &dev_attr_sess_boot_root.attr,
        &dev_attr_sess_boot_nic.attr,
        &dev_attr_sess_boot_target.attr,
        &dev_attr_priv_sess_recovery_tmo.attr,
        &dev_attr_priv_sess_state.attr,
        &dev_attr_priv_sess_target_state.attr,
        &dev_attr_priv_sess_creator.attr,
        &dev_attr_sess_chap_out_idx.attr,
        &dev_attr_sess_chap_in_idx.attr,
        &dev_attr_priv_sess_target_id.attr,
        &dev_attr_sess_auto_snd_tgt_disable.attr,
        &dev_attr_sess_discovery_session.attr,
        &dev_attr_sess_portal_type.attr,
        &dev_attr_sess_chap_auth.attr,
        &dev_attr_sess_discovery_logout.attr,
        &dev_attr_sess_bidi_chap.attr,
        &dev_attr_sess_discovery_auth_optional.attr,
        &dev_attr_sess_def_time2wait.attr,
        &dev_attr_sess_def_time2retain.attr,
        &dev_attr_sess_isid.attr,
        &dev_attr_sess_tsid.attr,
        &dev_attr_sess_def_taskmgmt_tmo.attr,
        &dev_attr_sess_discovery_parent_idx.attr,
        &dev_attr_sess_discovery_parent_type.attr,
        NULL,
};

static umode_t iscsi_session_attr_is_visible(struct kobject *kobj,
                                            struct attribute *attr, int i)
{
        struct device *cdev = container_of(kobj, struct device, kobj);
        struct iscsi_cls_session *session = transport_class_to_session(cdev);
        struct iscsi_transport *t = session->transport;
        int param;

        if (attr == &dev_attr_sess_initial_r2t.attr)
                param = ISCSI_PARAM_INITIAL_R2T_EN;
        else if (attr == &dev_attr_sess_max_outstanding_r2t.attr)
                param = ISCSI_PARAM_MAX_R2T;
        else if (attr == &dev_attr_sess_immediate_data.attr)
                param = ISCSI_PARAM_IMM_DATA_EN;
        else if (attr == &dev_attr_sess_first_burst_len.attr)
                param = ISCSI_PARAM_FIRST_BURST;
        else if (attr == &dev_attr_sess_max_burst_len.attr)
                param = ISCSI_PARAM_MAX_BURST;
        else if (attr == &dev_attr_sess_data_pdu_in_order.attr)
                param = ISCSI_PARAM_PDU_INORDER_EN;
        else if (attr == &dev_attr_sess_data_seq_in_order.attr)
                param = ISCSI_PARAM_DATASEQ_INORDER_EN;
        else if (attr == &dev_attr_sess_erl.attr)
                param = ISCSI_PARAM_ERL;
        else if (attr == &dev_attr_sess_targetname.attr)
                param = ISCSI_PARAM_TARGET_NAME;
        else if (attr == &dev_attr_sess_tpgt.attr)
                param = ISCSI_PARAM_TPGT;
        else if (attr == &dev_attr_sess_chap_in_idx.attr)
                param = ISCSI_PARAM_CHAP_IN_IDX;
        else if (attr == &dev_attr_sess_chap_out_idx.attr)
                param = ISCSI_PARAM_CHAP_OUT_IDX;
        else if (attr == &dev_attr_sess_password.attr)
                param = ISCSI_PARAM_USERNAME;
        else if (attr == &dev_attr_sess_password_in.attr)
                param = ISCSI_PARAM_USERNAME_IN;
        else if (attr == &dev_attr_sess_username.attr)
                param = ISCSI_PARAM_PASSWORD;
        else if (attr == &dev_attr_sess_username_in.attr)
                param = ISCSI_PARAM_PASSWORD_IN;
        else if (attr == &dev_attr_sess_fast_abort.attr)
                param = ISCSI_PARAM_FAST_ABORT;
        else if (attr == &dev_attr_sess_abort_tmo.attr)
                param = ISCSI_PARAM_ABORT_TMO;
        else if (attr == &dev_attr_sess_lu_reset_tmo.attr)
                param = ISCSI_PARAM_LU_RESET_TMO;
        else if (attr == &dev_attr_sess_tgt_reset_tmo.attr)
                param = ISCSI_PARAM_TGT_RESET_TMO;
        else if (attr == &dev_attr_sess_ifacename.attr)
                param = ISCSI_PARAM_IFACE_NAME;
        else if (attr == &dev_attr_sess_initiatorname.attr)
                param = ISCSI_PARAM_INITIATOR_NAME;
        else if (attr == &dev_attr_sess_targetalias.attr)
                param = ISCSI_PARAM_TARGET_ALIAS;
        else if (attr == &dev_attr_sess_boot_root.attr)
                param = ISCSI_PARAM_BOOT_ROOT;
        else if (attr == &dev_attr_sess_boot_nic.attr)
                param = ISCSI_PARAM_BOOT_NIC;
        else if (attr == &dev_attr_sess_boot_target.attr)
                param = ISCSI_PARAM_BOOT_TARGET;
        else if (attr == &dev_attr_sess_auto_snd_tgt_disable.attr)
                param = ISCSI_PARAM_AUTO_SND_TGT_DISABLE;
        else if (attr == &dev_attr_sess_discovery_session.attr)
                param = ISCSI_PARAM_DISCOVERY_SESS;
        else if (attr == &dev_attr_sess_portal_type.attr)
                param = ISCSI_PARAM_PORTAL_TYPE;
        else if (attr == &dev_attr_sess_chap_auth.attr)
                param = ISCSI_PARAM_CHAP_AUTH_EN;
        else if (attr == &dev_attr_sess_discovery_logout.attr)
                param = ISCSI_PARAM_DISCOVERY_LOGOUT_EN;
        else if (attr == &dev_attr_sess_bidi_chap.attr)
                param = ISCSI_PARAM_BIDI_CHAP_EN;
        else if (attr == &dev_attr_sess_discovery_auth_optional.attr)
                param = ISCSI_PARAM_DISCOVERY_AUTH_OPTIONAL;
        else if (attr == &dev_attr_sess_def_time2wait.attr)
                param = ISCSI_PARAM_DEF_TIME2WAIT;
        else if (attr == &dev_attr_sess_def_time2retain.attr)
                param = ISCSI_PARAM_DEF_TIME2RETAIN;
        else if (attr == &dev_attr_sess_isid.attr)
                param = ISCSI_PARAM_ISID;
        else if (attr == &dev_attr_sess_tsid.attr)
                param = ISCSI_PARAM_TSID;
        else if (attr == &dev_attr_sess_def_taskmgmt_tmo.attr)
                param = ISCSI_PARAM_DEF_TASKMGMT_TMO;
        else if (attr == &dev_attr_sess_discovery_parent_idx.attr)
                param = ISCSI_PARAM_DISCOVERY_PARENT_IDX;
        else if (attr == &dev_attr_sess_discovery_parent_type.attr)
                param = ISCSI_PARAM_DISCOVERY_PARENT_TYPE;
        else if (attr == &dev_attr_priv_sess_recovery_tmo.attr)
                return S_IRUGO | S_IWUSR;
        else if (attr == &dev_attr_priv_sess_state.attr)
                return S_IRUGO;
        else if (attr == &dev_attr_priv_sess_target_state.attr)
                return S_IRUGO;
        else if (attr == &dev_attr_priv_sess_creator.attr)
                return S_IRUGO;
        else if (attr == &dev_attr_priv_sess_target_id.attr)
                return S_IRUGO;
        else {
                WARN_ONCE(1, "Invalid session attr");
                return 0;
        }

        return t->attr_is_visible(ISCSI_PARAM, param);
}

static struct attribute_group iscsi_session_group = {
        .attrs = iscsi_session_attrs,
        .is_visible = iscsi_session_attr_is_visible,
};

/*
 * iSCSI host attrs
 */
#define iscsi_host_attr_show(param)                                        \
static ssize_t                                                                \
show_host_param_##param(struct device *dev,                                 \
                        struct device_attribute *attr, char *buf)        \
{                                                                        \
        struct Scsi_Host *shost = transport_class_to_shost(dev);        \
        struct iscsi_internal *priv = to_iscsi_internal(shost->transportt); \
        return priv->iscsi_transport->get_host_param(shost, param, buf); \
}

#define iscsi_host_attr(field, param)                                        \
        iscsi_host_attr_show(param)                                        \
static ISCSI_CLASS_ATTR(host, field, S_IRUGO, show_host_param_##param,        \
                        NULL);

iscsi_host_attr(netdev, ISCSI_HOST_PARAM_NETDEV_NAME);
iscsi_host_attr(hwaddress, ISCSI_HOST_PARAM_HWADDRESS);
iscsi_host_attr(ipaddress, ISCSI_HOST_PARAM_IPADDRESS);
iscsi_host_attr(initiatorname, ISCSI_HOST_PARAM_INITIATOR_NAME);
iscsi_host_attr(port_state, ISCSI_HOST_PARAM_PORT_STATE);
iscsi_host_attr(port_speed, ISCSI_HOST_PARAM_PORT_SPEED);

static struct attribute *iscsi_host_attrs[] = {
        &dev_attr_host_netdev.attr,
        &dev_attr_host_hwaddress.attr,
        &dev_attr_host_ipaddress.attr,
        &dev_attr_host_initiatorname.attr,
        &dev_attr_host_port_state.attr,
        &dev_attr_host_port_speed.attr,
        NULL,
};

static umode_t iscsi_host_attr_is_visible(struct kobject *kobj,
                                         struct attribute *attr, int i)
{
        struct device *cdev = container_of(kobj, struct device, kobj);
        struct Scsi_Host *shost = transport_class_to_shost(cdev);
        struct iscsi_internal *priv = to_iscsi_internal(shost->transportt);
        int param;

        if (attr == &dev_attr_host_netdev.attr)
                param = ISCSI_HOST_PARAM_NETDEV_NAME;
        else if (attr == &dev_attr_host_hwaddress.attr)
                param = ISCSI_HOST_PARAM_HWADDRESS;
        else if (attr == &dev_attr_host_ipaddress.attr)
                param = ISCSI_HOST_PARAM_IPADDRESS;
        else if (attr == &dev_attr_host_initiatorname.attr)
                param = ISCSI_HOST_PARAM_INITIATOR_NAME;
        else if (attr == &dev_attr_host_port_state.attr)
                param = ISCSI_HOST_PARAM_PORT_STATE;
        else if (attr == &dev_attr_host_port_speed.attr)
                param = ISCSI_HOST_PARAM_PORT_SPEED;
        else {
                WARN_ONCE(1, "Invalid host attr");
                return 0;
        }

        return priv->iscsi_transport->attr_is_visible(ISCSI_HOST_PARAM, param);
}

static struct attribute_group iscsi_host_group = {
        .attrs = iscsi_host_attrs,
        .is_visible = iscsi_host_attr_is_visible,
};

/* convert iscsi_port_speed values to ascii string name */
static const struct {
        enum iscsi_port_speed        value;
        char                        *name;
} iscsi_port_speed_names[] = {
        {ISCSI_PORT_SPEED_UNKNOWN,        "Unknown" },
        {ISCSI_PORT_SPEED_10MBPS,        "10 Mbps" },
        {ISCSI_PORT_SPEED_100MBPS,        "100 Mbps" },
        {ISCSI_PORT_SPEED_1GBPS,        "1 Gbps" },
        {ISCSI_PORT_SPEED_10GBPS,        "10 Gbps" },
        {ISCSI_PORT_SPEED_25GBPS,       "25 Gbps" },
        {ISCSI_PORT_SPEED_40GBPS,       "40 Gbps" },
};

char *iscsi_get_port_speed_name(struct Scsi_Host *shost)
{
        int i;
        char *speed = "Unknown!";
        struct iscsi_cls_host *ihost = shost->shost_data;
        uint32_t port_speed = ihost->port_speed;

        for (i = 0; i < ARRAY_SIZE(iscsi_port_speed_names); i++) {
                if (iscsi_port_speed_names[i].value & port_speed) {
                        speed = iscsi_port_speed_names[i].name;
                        break;
                }
        }
        return speed;
}
EXPORT_SYMBOL_GPL(iscsi_get_port_speed_name);

/* convert iscsi_port_state values to ascii string name */
static const struct {
        enum iscsi_port_state        value;
        char                        *name;
} iscsi_port_state_names[] = {
        {ISCSI_PORT_STATE_DOWN,                "LINK DOWN" },
        {ISCSI_PORT_STATE_UP,                "LINK UP" },
};

char *iscsi_get_port_state_name(struct Scsi_Host *shost)
{
        int i;
        char *state = "Unknown!";
        struct iscsi_cls_host *ihost = shost->shost_data;
        uint32_t port_state = ihost->port_state;

        for (i = 0; i < ARRAY_SIZE(iscsi_port_state_names); i++) {
                if (iscsi_port_state_names[i].value & port_state) {
                        state = iscsi_port_state_names[i].name;
                        break;
                }
        }
        return state;
}
EXPORT_SYMBOL_GPL(iscsi_get_port_state_name);

static int iscsi_session_match(struct attribute_container *cont,
                           struct device *dev)
{
        struct iscsi_cls_session *session;
        struct Scsi_Host *shost;
        struct iscsi_internal *priv;

        if (!iscsi_is_session_dev(dev))
                return 0;

        session = iscsi_dev_to_session(dev);
        shost = iscsi_session_to_shost(session);
        if (!shost->transportt)
                return 0;

        priv = to_iscsi_internal(shost->transportt);
        if (priv->session_cont.ac.class != &iscsi_session_class.class)
                return 0;

        return &priv->session_cont.ac == cont;
}

static int iscsi_conn_match(struct attribute_container *cont,
                           struct device *dev)
{
        struct iscsi_cls_session *session;
        struct iscsi_cls_conn *conn;
        struct Scsi_Host *shost;
        struct iscsi_internal *priv;

        if (!iscsi_is_conn_dev(dev))
                return 0;

        conn = iscsi_dev_to_conn(dev);
        session = iscsi_dev_to_session(conn->dev.parent);
        shost = iscsi_session_to_shost(session);

        if (!shost->transportt)
                return 0;

        priv = to_iscsi_internal(shost->transportt);
        if (priv->conn_cont.ac.class != &iscsi_connection_class.class)
                return 0;

        return &priv->conn_cont.ac == cont;
}

static int iscsi_host_match(struct attribute_container *cont,
                            struct device *dev)
{
        struct Scsi_Host *shost;
        struct iscsi_internal *priv;

        if (!scsi_is_host_device(dev))
                return 0;

        shost = dev_to_shost(dev);
        if (!shost->transportt  ||
            shost->transportt->host_attrs.ac.class != &iscsi_host_class.class)
                return 0;

        priv = to_iscsi_internal(shost->transportt);
        return &priv->t.host_attrs.ac == cont;
}

struct scsi_transport_template *
iscsi_register_transport(struct iscsi_transport *tt)
{
        struct iscsi_internal *priv;
        unsigned long flags;
        int err;

        BUG_ON(!tt);
        WARN_ON(tt->ep_disconnect && !tt->unbind_conn);

        priv = iscsi_if_transport_lookup(tt);
        if (priv)
                return NULL;

        priv = kzalloc_obj(*priv);
        if (!priv)
                return NULL;
        INIT_LIST_HEAD(&priv->list);
        priv->iscsi_transport = tt;
        priv->t.user_scan = iscsi_user_scan;

        priv->dev.class = &iscsi_transport_class;
        dev_set_name(&priv->dev, "%s", tt->name);
        err = device_register(&priv->dev);
        if (err)
                goto put_dev;

        err = sysfs_create_group(&priv->dev.kobj, &iscsi_transport_group);
        if (err)
                goto unregister_dev;

        /* host parameters */
        priv->t.host_attrs.ac.class = &iscsi_host_class.class;
        priv->t.host_attrs.ac.match = iscsi_host_match;
        priv->t.host_attrs.ac.grp = &iscsi_host_group;
        priv->t.host_size = sizeof(struct iscsi_cls_host);
        transport_container_register(&priv->t.host_attrs);

        /* connection parameters */
        priv->conn_cont.ac.class = &iscsi_connection_class.class;
        priv->conn_cont.ac.match = iscsi_conn_match;
        priv->conn_cont.ac.grp = &iscsi_conn_group;
        transport_container_register(&priv->conn_cont);

        /* session parameters */
        priv->session_cont.ac.class = &iscsi_session_class.class;
        priv->session_cont.ac.match = iscsi_session_match;
        priv->session_cont.ac.grp = &iscsi_session_group;
        transport_container_register(&priv->session_cont);

        spin_lock_irqsave(&iscsi_transport_lock, flags);
        list_add(&priv->list, &iscsi_transports);
        spin_unlock_irqrestore(&iscsi_transport_lock, flags);

        printk(KERN_NOTICE "iscsi: registered transport (%s)\n", tt->name);
        return &priv->t;

unregister_dev:
        device_unregister(&priv->dev);
        return NULL;
put_dev:
        put_device(&priv->dev);
        return NULL;
}
EXPORT_SYMBOL_GPL(iscsi_register_transport);

void iscsi_unregister_transport(struct iscsi_transport *tt)
{
        struct iscsi_internal *priv;
        unsigned long flags;

        BUG_ON(!tt);

        mutex_lock(&rx_queue_mutex);

        priv = iscsi_if_transport_lookup(tt);
        BUG_ON (!priv);

        spin_lock_irqsave(&iscsi_transport_lock, flags);
        list_del(&priv->list);
        spin_unlock_irqrestore(&iscsi_transport_lock, flags);

        transport_container_unregister(&priv->conn_cont);
        transport_container_unregister(&priv->session_cont);
        transport_container_unregister(&priv->t.host_attrs);

        sysfs_remove_group(&priv->dev.kobj, &iscsi_transport_group);
        device_unregister(&priv->dev);
        mutex_unlock(&rx_queue_mutex);
}
EXPORT_SYMBOL_GPL(iscsi_unregister_transport);

void iscsi_dbg_trace(void (*trace)(struct device *dev, struct va_format *),
                     struct device *dev, const char *fmt, ...)
{
        struct va_format vaf;
        va_list args;

        va_start(args, fmt);
        vaf.fmt = fmt;
        vaf.va = &args;
        trace(dev, &vaf);
        va_end(args);
}
EXPORT_SYMBOL_GPL(iscsi_dbg_trace);

static __init int iscsi_transport_init(void)
{
        int err;
        struct netlink_kernel_cfg cfg = {
                .groups        = 1,
                .input        = iscsi_if_rx,
        };
        printk(KERN_INFO "Loading iSCSI transport class v%s.\n",
                ISCSI_TRANSPORT_VERSION);

        atomic_set(&iscsi_session_nr, 0);

        err = class_register(&iscsi_transport_class);
        if (err)
                return err;

        err = class_register(&iscsi_endpoint_class);
        if (err)
                goto unregister_transport_class;

        err = class_register(&iscsi_iface_class);
        if (err)
                goto unregister_endpoint_class;

        err = transport_class_register(&iscsi_host_class);
        if (err)
                goto unregister_iface_class;

        err = transport_class_register(&iscsi_connection_class);
        if (err)
                goto unregister_host_class;

        err = transport_class_register(&iscsi_session_class);
        if (err)
                goto unregister_conn_class;

        err = bus_register(&iscsi_flashnode_bus);
        if (err)
                goto unregister_session_class;

        nls = netlink_kernel_create(&init_net, NETLINK_ISCSI, &cfg);
        if (!nls) {
                err = -ENOBUFS;
                goto unregister_flashnode_bus;
        }

        iscsi_conn_cleanup_workq = alloc_workqueue("%s",
                        WQ_SYSFS | WQ_MEM_RECLAIM | WQ_UNBOUND, 0,
                        "iscsi_conn_cleanup");
        if (!iscsi_conn_cleanup_workq) {
                err = -ENOMEM;
                goto release_nls;
        }

        return 0;

release_nls:
        netlink_kernel_release(nls);
unregister_flashnode_bus:
        bus_unregister(&iscsi_flashnode_bus);
unregister_session_class:
        transport_class_unregister(&iscsi_session_class);
unregister_conn_class:
        transport_class_unregister(&iscsi_connection_class);
unregister_host_class:
        transport_class_unregister(&iscsi_host_class);
unregister_iface_class:
        class_unregister(&iscsi_iface_class);
unregister_endpoint_class:
        class_unregister(&iscsi_endpoint_class);
unregister_transport_class:
        class_unregister(&iscsi_transport_class);
        return err;
}

static void __exit iscsi_transport_exit(void)
{
        destroy_workqueue(iscsi_conn_cleanup_workq);
        netlink_kernel_release(nls);
        bus_unregister(&iscsi_flashnode_bus);
        transport_class_unregister(&iscsi_connection_class);
        transport_class_unregister(&iscsi_session_class);
        transport_class_unregister(&iscsi_host_class);
        class_unregister(&iscsi_endpoint_class);
        class_unregister(&iscsi_iface_class);
        class_unregister(&iscsi_transport_class);
}

module_init(iscsi_transport_init);
module_exit(iscsi_transport_exit);

MODULE_AUTHOR("Mike Christie <michaelc@cs.wisc.edu>, "
              "Dmitry Yusupov <dmitry_yus@yahoo.com>, "
              "Alex Aizman <itn780@yahoo.com>");
MODULE_DESCRIPTION("iSCSI Transport Interface");
MODULE_LICENSE("GPL");
MODULE_VERSION(ISCSI_TRANSPORT_VERSION);
MODULE_ALIAS_NET_PF_PROTO(PF_NETLINK, NETLINK_ISCSI);





































































    1 


    1 
    1 














    4 
    2 


    3 
    4 
    3 
    4 

    2 





    4 





    3 
    4 

    4 
    4 






    3 






    4 
    2 





    3 


    3 
    3 
    3 














    3 
    3 





    3 

















    5 
    1 



    3 
    4 




























    4 
    4 


    4 
    2 



    4 



    4 




    4 



    4 



    2 






    3 

    3 

    4 






    2 










    5 
    4 




    3 
    3 









































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
// SPDX-License-Identifier: GPL-2.0-only
/*
 * H-TCP congestion control. The algorithm is detailed in:
 * R.N.Shorten, D.J.Leith:
 *   "H-TCP: TCP for high-speed and long-distance networks"
 *   Proc. PFLDnet, Argonne, 2004.
 * https://www.hamilton.ie/net/htcp3.pdf
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <net/tcp.h>

#define ALPHA_BASE        (1<<7)        /* 1.0 with shift << 7 */
#define BETA_MIN        (1<<6)        /* 0.5 with shift << 7 */
#define BETA_MAX        102        /* 0.8 with shift << 7 */

static int use_rtt_scaling __read_mostly = 1;
module_param(use_rtt_scaling, int, 0644);
MODULE_PARM_DESC(use_rtt_scaling, "turn on/off RTT scaling");

static int use_bandwidth_switch __read_mostly = 1;
module_param(use_bandwidth_switch, int, 0644);
MODULE_PARM_DESC(use_bandwidth_switch, "turn on/off bandwidth switcher");

struct htcp {
        u32        alpha;                /* Fixed point arith, << 7 */
        u8        beta;           /* Fixed point arith, << 7 */
        u8        modeswitch;        /* Delay modeswitch
                                   until we had at least one congestion event */
        u16        pkts_acked;
        u32        packetcount;
        u32        minRTT;
        u32        maxRTT;
        u32        last_cong;        /* Time since last congestion event end */
        u32        undo_last_cong;

        u32        undo_maxRTT;
        u32        undo_old_maxB;

        /* Bandwidth estimation */
        u32        minB;
        u32        maxB;
        u32        old_maxB;
        u32        Bi;
        u32        lasttime;
};

static inline u32 htcp_cong_time(const struct htcp *ca)
{
        return jiffies - ca->last_cong;
}

static inline u32 htcp_ccount(const struct htcp *ca)
{
        return htcp_cong_time(ca) / ca->minRTT;
}

static inline void htcp_reset(struct htcp *ca)
{
        ca->undo_last_cong = ca->last_cong;
        ca->undo_maxRTT = ca->maxRTT;
        ca->undo_old_maxB = ca->old_maxB;

        ca->last_cong = jiffies;
}

static u32 htcp_cwnd_undo(struct sock *sk)
{
        struct htcp *ca = inet_csk_ca(sk);

        if (ca->undo_last_cong) {
                ca->last_cong = ca->undo_last_cong;
                ca->maxRTT = ca->undo_maxRTT;
                ca->old_maxB = ca->undo_old_maxB;
                ca->undo_last_cong = 0;
        }

        return tcp_reno_undo_cwnd(sk);
}

static inline void measure_rtt(struct sock *sk, u32 srtt)
{
        const struct inet_connection_sock *icsk = inet_csk(sk);
        struct htcp *ca = inet_csk_ca(sk);

        /* keep track of minimum RTT seen so far, minRTT is zero at first */
        if (ca->minRTT > srtt || !ca->minRTT)
                ca->minRTT = srtt;

        /* max RTT */
        if (icsk->icsk_ca_state == TCP_CA_Open) {
                if (ca->maxRTT < ca->minRTT)
                        ca->maxRTT = ca->minRTT;
                if (ca->maxRTT < srtt &&
                    srtt <= ca->maxRTT + msecs_to_jiffies(20))
                        ca->maxRTT = srtt;
        }
}

static void measure_achieved_throughput(struct sock *sk,
                                        const struct ack_sample *sample)
{
        const struct inet_connection_sock *icsk = inet_csk(sk);
        const struct tcp_sock *tp = tcp_sk(sk);
        struct htcp *ca = inet_csk_ca(sk);
        u32 now = tcp_jiffies32;

        if (icsk->icsk_ca_state == TCP_CA_Open)
                ca->pkts_acked = sample->pkts_acked;

        if (sample->rtt_us > 0)
                measure_rtt(sk, usecs_to_jiffies(sample->rtt_us));

        if (!use_bandwidth_switch)
                return;

        /* achieved throughput calculations */
        if (!((1 << icsk->icsk_ca_state) & (TCPF_CA_Open | TCPF_CA_Disorder))) {
                ca->packetcount = 0;
                ca->lasttime = now;
                return;
        }

        ca->packetcount += sample->pkts_acked;

        if (ca->packetcount >= tcp_snd_cwnd(tp) - (ca->alpha >> 7 ? : 1) &&
            now - ca->lasttime >= ca->minRTT &&
            ca->minRTT > 0) {
                __u32 cur_Bi = ca->packetcount * HZ / (now - ca->lasttime);

                if (htcp_ccount(ca) <= 3) {
                        /* just after backoff */
                        ca->minB = ca->maxB = ca->Bi = cur_Bi;
                } else {
                        ca->Bi = (3 * ca->Bi + cur_Bi) / 4;
                        if (ca->Bi > ca->maxB)
                                ca->maxB = ca->Bi;
                        if (ca->minB > ca->maxB)
                                ca->minB = ca->maxB;
                }
                ca->packetcount = 0;
                ca->lasttime = now;
        }
}

static inline void htcp_beta_update(struct htcp *ca, u32 minRTT, u32 maxRTT)
{
        if (use_bandwidth_switch) {
                u32 maxB = ca->maxB;
                u32 old_maxB = ca->old_maxB;

                ca->old_maxB = ca->maxB;
                if (!between(5 * maxB, 4 * old_maxB, 6 * old_maxB)) {
                        ca->beta = BETA_MIN;
                        ca->modeswitch = 0;
                        return;
                }
        }

        if (ca->modeswitch && minRTT > msecs_to_jiffies(10) && maxRTT) {
                ca->beta = (minRTT << 7) / maxRTT;
                if (ca->beta < BETA_MIN)
                        ca->beta = BETA_MIN;
                else if (ca->beta > BETA_MAX)
                        ca->beta = BETA_MAX;
        } else {
                ca->beta = BETA_MIN;
                ca->modeswitch = 1;
        }
}

static inline void htcp_alpha_update(struct htcp *ca)
{
        u32 minRTT = ca->minRTT;
        u32 factor = 1;
        u32 diff = htcp_cong_time(ca);

        if (diff > HZ) {
                diff -= HZ;
                factor = 1 + (10 * diff + ((diff / 2) * (diff / 2) / HZ)) / HZ;
        }

        if (use_rtt_scaling && minRTT) {
                u32 scale = (HZ << 3) / (10 * minRTT);

                /* clamping ratio to interval [0.5,10]<<3 */
                scale = clamp(scale, 1U << 2, 10U << 3);
                factor = (factor << 3) / scale;
                if (!factor)
                        factor = 1;
        }

        ca->alpha = 2 * factor * ((1 << 7) - ca->beta);
        if (!ca->alpha)
                ca->alpha = ALPHA_BASE;
}

/*
 * After we have the rtt data to calculate beta, we'd still prefer to wait one
 * rtt before we adjust our beta to ensure we are working from a consistent
 * data.
 *
 * This function should be called when we hit a congestion event since only at
 * that point do we really have a real sense of maxRTT (the queues en route
 * were getting just too full now).
 */
static void htcp_param_update(struct sock *sk)
{
        struct htcp *ca = inet_csk_ca(sk);
        u32 minRTT = ca->minRTT;
        u32 maxRTT = ca->maxRTT;

        htcp_beta_update(ca, minRTT, maxRTT);
        htcp_alpha_update(ca);

        /* add slowly fading memory for maxRTT to accommodate routing changes */
        if (minRTT > 0 && maxRTT > minRTT)
                ca->maxRTT = minRTT + ((maxRTT - minRTT) * 95) / 100;
}

static u32 htcp_recalc_ssthresh(struct sock *sk)
{
        const struct tcp_sock *tp = tcp_sk(sk);
        const struct htcp *ca = inet_csk_ca(sk);

        htcp_param_update(sk);
        return max((tcp_snd_cwnd(tp) * ca->beta) >> 7, 2U);
}

static void htcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
{
        struct tcp_sock *tp = tcp_sk(sk);
        struct htcp *ca = inet_csk_ca(sk);

        if (!tcp_is_cwnd_limited(sk))
                return;

        if (tcp_in_slow_start(tp))
                tcp_slow_start(tp, acked);
        else {
                /* In dangerous area, increase slowly.
                 * In theory this is tp->snd_cwnd += alpha / tp->snd_cwnd
                 */
                if ((tp->snd_cwnd_cnt * ca->alpha)>>7 >= tcp_snd_cwnd(tp)) {
                        if (tcp_snd_cwnd(tp) < tp->snd_cwnd_clamp)
                                tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1);
                        tp->snd_cwnd_cnt = 0;
                        htcp_alpha_update(ca);
                } else
                        tp->snd_cwnd_cnt += ca->pkts_acked;

                ca->pkts_acked = 1;
        }
}

static void htcp_init(struct sock *sk)
{
        struct htcp *ca = inet_csk_ca(sk);

        memset(ca, 0, sizeof(struct htcp));
        ca->alpha = ALPHA_BASE;
        ca->beta = BETA_MIN;
        ca->pkts_acked = 1;
        ca->last_cong = jiffies;
}

static void htcp_state(struct sock *sk, u8 new_state)
{
        switch (new_state) {
        case TCP_CA_Open:
                {
                        struct htcp *ca = inet_csk_ca(sk);

                        if (ca->undo_last_cong) {
                                ca->last_cong = jiffies;
                                ca->undo_last_cong = 0;
                        }
                }
                break;
        case TCP_CA_CWR:
        case TCP_CA_Recovery:
        case TCP_CA_Loss:
                htcp_reset(inet_csk_ca(sk));
                break;
        }
}

static struct tcp_congestion_ops htcp __read_mostly = {
        .init                = htcp_init,
        .ssthresh        = htcp_recalc_ssthresh,
        .cong_avoid        = htcp_cong_avoid,
        .set_state        = htcp_state,
        .undo_cwnd        = htcp_cwnd_undo,
        .pkts_acked        = measure_achieved_throughput,
        .owner                = THIS_MODULE,
        .name                = "htcp",
};

static int __init htcp_register(void)
{
        BUILD_BUG_ON(sizeof(struct htcp) > ICSK_CA_PRIV_SIZE);
        BUILD_BUG_ON(BETA_MIN >= BETA_MAX);
        return tcp_register_congestion_control(&htcp);
}

static void __exit htcp_unregister(void)
{
        tcp_unregister_congestion_control(&htcp);
}

module_init(htcp_register);
module_exit(htcp_unregister);

MODULE_AUTHOR("Baruch Even");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("H-TCP");
































































































































































































































































































































































































































































































































































































































































































   37 



























   37 







































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
// SPDX-License-Identifier: GPL-2.0-only
/*
 * AppArmor security module
 *
 * This file contains AppArmor dfa based regular expression matching engine
 *
 * Copyright (C) 1998-2008 Novell/SUSE
 * Copyright 2009-2012 Canonical Ltd.
 */

#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/err.h>
#include <linux/kref.h>
#include <linux/unaligned.h>

#include "include/lib.h"
#include "include/match.h"

#define base_idx(X) ((X) & 0xffffff)

/**
 * unpack_table - unpack a dfa table (one of accept, default, base, next check)
 * @blob: data to unpack (NOT NULL)
 * @bsize: size of blob
 *
 * Returns: pointer to table else ERR_PTR on failure
 *
 * NOTE: must be freed by kvfree (not kfree)
 */
static struct table_header *unpack_table(const char *blob, size_t bsize)
{
        struct table_header *table = ERR_PTR(-EPROTO);
        struct table_header th;
        size_t tsize;

        if (bsize < sizeof(struct table_header))
                goto out;

        /* loaded td_id's start at 1, subtract 1 now to avoid doing
         * it every time we use td_id as an index
         */
        th.td_id = get_unaligned_be16(blob) - 1;
        if (th.td_id > YYTD_ID_MAX)
                goto out;
        th.td_flags = get_unaligned_be16(blob + 2);
        th.td_lolen = get_unaligned_be32(blob + 8);
        blob += sizeof(struct table_header);

        if (!(th.td_flags == YYTD_DATA16 || th.td_flags == YYTD_DATA32 ||
              th.td_flags == YYTD_DATA8))
                goto out;

        /* if we have a table it must have some entries */
        if (th.td_lolen == 0)
                goto out;
        tsize = table_size(th.td_lolen, th.td_flags);
        if (bsize < tsize)
                goto out;

        table = kvzalloc(tsize, GFP_KERNEL);
        if (table) {
                table->td_id = th.td_id;
                table->td_flags = th.td_flags;
                table->td_lolen = th.td_lolen;
                if (th.td_flags == YYTD_DATA8)
                        memcpy(table->td_data, blob, th.td_lolen);
                else if (th.td_flags == YYTD_DATA16)
                        UNPACK_ARRAY(table->td_data, blob, th.td_lolen,
                                     u16, __be16, get_unaligned_be16);
                else if (th.td_flags == YYTD_DATA32)
                        UNPACK_ARRAY(table->td_data, blob, th.td_lolen,
                                     u32, __be32, get_unaligned_be32);
                else {
                        kvfree(table);
                        table = ERR_PTR(-EPROTO);
                        goto out;
                }
                /* if table was vmalloced make sure the page tables are synced
                 * before it is used, as it goes live to all cpus.
                 */
                if (is_vmalloc_addr(table))
                        vm_unmap_aliases();
        } else
                table = ERR_PTR(-ENOMEM);

out:
        return table;
}

/**
 * verify_table_headers - verify that the tables headers are as expected
 * @tables: array of dfa tables to check (NOT NULL)
 * @flags: flags controlling what type of accept table are acceptable
 *
 * Assumes dfa has gone through the first pass verification done by unpacking
 * NOTE: this does not valid accept table values
 *
 * Returns: %0 else error code on failure to verify
 */
static int verify_table_headers(struct table_header **tables, int flags)
{
        size_t state_count, trans_count;
        int error = -EPROTO;

        /* check that required tables exist */
        if (!(tables[YYTD_ID_DEF] && tables[YYTD_ID_BASE] &&
              tables[YYTD_ID_NXT] && tables[YYTD_ID_CHK]))
                goto out;

        /* accept.size == default.size == base.size */
        state_count = tables[YYTD_ID_BASE]->td_lolen;
        if (ACCEPT1_FLAGS(flags)) {
                if (!tables[YYTD_ID_ACCEPT])
                        goto out;
                if (state_count != tables[YYTD_ID_ACCEPT]->td_lolen)
                        goto out;
        }
        if (ACCEPT2_FLAGS(flags)) {
                if (!tables[YYTD_ID_ACCEPT2])
                        goto out;
                if (state_count != tables[YYTD_ID_ACCEPT2]->td_lolen)
                        goto out;
        }
        if (state_count != tables[YYTD_ID_DEF]->td_lolen)
                goto out;

        /* next.size == chk.size */
        trans_count = tables[YYTD_ID_NXT]->td_lolen;
        if (trans_count != tables[YYTD_ID_CHK]->td_lolen)
                goto out;

        /* if equivalence classes then its table size must be 256 */
        if (tables[YYTD_ID_EC] && tables[YYTD_ID_EC]->td_lolen != 256)
                goto out;

        error = 0;
out:
        return error;
}

/**
 * verify_dfa - verify that transitions and states in the tables are in bounds.
 * @dfa: dfa to test  (NOT NULL)
 *
 * Assumes dfa has gone through the first pass verification done by unpacking
 * NOTE: this does not valid accept table values
 *
 * Returns: %0 else error code on failure to verify
 */
static int verify_dfa(const struct aa_dfa *dfa)
{
        size_t i, state_count, trans_count;
        int error = -EPROTO;

        state_count = dfa->tables[YYTD_ID_BASE]->td_lolen;
        trans_count = dfa->tables[YYTD_ID_NXT]->td_lolen;
        if (state_count < 2)
                goto out;
        for (i = 0; i < state_count; i++) {
                if (DEFAULT_TABLE(dfa)[i] >= state_count) {
                        pr_err("AppArmor DFA default state out of bounds");
                        goto out;
                }
                if (BASE_TABLE(dfa)[i] & MATCH_FLAGS_INVALID) {
                        pr_err("AppArmor DFA state with invalid match flags");
                        goto out;
                }
                if ((BASE_TABLE(dfa)[i] & MATCH_FLAG_DIFF_ENCODE)) {
                        if (!(dfa->flags & YYTH_FLAG_DIFF_ENCODE)) {
                                pr_err("AppArmor DFA diff encoded transition state without header flag");
                                goto out;
                        }
                }
                if ((BASE_TABLE(dfa)[i] & MATCH_FLAG_OOB_TRANSITION)) {
                        if (base_idx(BASE_TABLE(dfa)[i]) < dfa->max_oob) {
                                pr_err("AppArmor DFA out of bad transition out of range");
                                goto out;
                        }
                        if (!(dfa->flags & YYTH_FLAG_OOB_TRANS)) {
                                pr_err("AppArmor DFA out of bad transition state without header flag");
                                goto out;
                        }
                }
                if (base_idx(BASE_TABLE(dfa)[i]) + 255 >= trans_count) {
                        pr_err("AppArmor DFA next/check upper bounds error\n");
                        goto out;
                }
        }

        for (i = 0; i < trans_count; i++) {
                if (NEXT_TABLE(dfa)[i] >= state_count)
                        goto out;
                if (CHECK_TABLE(dfa)[i] >= state_count)
                        goto out;
        }

        /* Now that all the other tables are verified, verify diffencoding */
        for (i = 0; i < state_count; i++) {
                size_t j, k;

                for (j = i;
                     ((BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE) &&
                      !(BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE_VERIFIED));
                     j = k) {
                        if (BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE)
                                /* loop in current chain */
                                goto out;
                        k = DEFAULT_TABLE(dfa)[j];
                        if (j == k)
                                /* self loop */
                                goto out;
                        BASE_TABLE(dfa)[j] |= MARK_DIFF_ENCODE;
                }
                /* move mark to verified */
                for (j = i;
                     (BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE);
                     j = k) {
                        k = DEFAULT_TABLE(dfa)[j];
                        if (j < i)
                                /* jumps to state/chain that has been
                                 * verified
                                 */
                                break;
                        BASE_TABLE(dfa)[j] &= ~MARK_DIFF_ENCODE;
                        BASE_TABLE(dfa)[j] |= MARK_DIFF_ENCODE_VERIFIED;
                }
        }
        error = 0;

out:
        return error;
}

/**
 * dfa_free - free a dfa allocated by aa_dfa_unpack
 * @dfa: the dfa to free  (MAYBE NULL)
 *
 * Requires: reference count to dfa == 0
 */
static void dfa_free(struct aa_dfa *dfa)
{
        if (dfa) {
                int i;

                for (i = 0; i < ARRAY_SIZE(dfa->tables); i++) {
                        kvfree(dfa->tables[i]);
                        dfa->tables[i] = NULL;
                }
                kfree(dfa);
        }
}

/**
 * aa_dfa_free_kref - free aa_dfa by kref (called by aa_put_dfa)
 * @kref: kref callback for freeing of a dfa  (NOT NULL)
 */
void aa_dfa_free_kref(struct kref *kref)
{
        struct aa_dfa *dfa = container_of(kref, struct aa_dfa, count);
        dfa_free(dfa);
}



/**
 * remap_data16_to_data32 - remap u16 @old table to a u32 based table
 * @old: table to remap
 *
 * Returns: new table with u32 entries instead of u16.
 *
 * Note: will free @old so caller does not have to
 */
static struct table_header *remap_data16_to_data32(struct table_header *old)
{
        struct table_header *new;
        size_t tsize;
        u32 i;

        tsize = table_size(old->td_lolen, YYTD_DATA32);
        new = kvzalloc(tsize, GFP_KERNEL);
        if (!new) {
                kvfree(old);
                return NULL;
        }
        new->td_id = old->td_id;
        new->td_flags = YYTD_DATA32;
        new->td_lolen = old->td_lolen;

        for (i = 0; i < old->td_lolen; i++)
                TABLE_DATAU32(new)[i] = (u32) TABLE_DATAU16(old)[i];

        kvfree(old);
        if (is_vmalloc_addr(new))
                vm_unmap_aliases();

        return new;
}

/**
 * aa_dfa_unpack - unpack the binary tables of a serialized dfa
 * @blob: aligned serialized stream of data to unpack  (NOT NULL)
 * @size: size of data to unpack
 * @flags: flags controlling what type of accept tables are acceptable
 *
 * Unpack a dfa that has been serialized.  To find information on the dfa
 * format look in Documentation/admin-guide/LSM/apparmor.rst
 * Assumes the dfa @blob stream has been aligned on a 8 byte boundary
 *
 * Returns: an unpacked dfa ready for matching or ERR_PTR on failure
 */
struct aa_dfa *aa_dfa_unpack(const void *blob, size_t size, int flags)
{
        int hsize;
        int error = -ENOMEM;
        const char *data = blob;
        struct table_header *table = NULL;
        struct aa_dfa *dfa = kzalloc_obj(struct aa_dfa);
        if (!dfa)
                goto fail;

        kref_init(&dfa->count);

        error = -EPROTO;

        /* get dfa table set header */
        if (size < sizeof(struct table_set_header))
                goto fail;

        if (get_unaligned_be32(data) != YYTH_MAGIC)
                goto fail;

        hsize = get_unaligned_be32(data + 4);
        if (size < hsize)
                goto fail;

        dfa->flags = get_unaligned_be16(data + 12);
        if (dfa->flags & ~(YYTH_FLAGS))
                goto fail;

        /*
         * TODO: needed for dfa to support more than 1 oob
         * if (dfa->flags & YYTH_FLAGS_OOB_TRANS) {
         *        if (hsize < 16 + 4)
         *                goto fail;
         *        dfa->max_oob = get_unaligned_be32(data + 16);
         *        if (dfa->max <= MAX_OOB_SUPPORTED) {
         *                pr_err("AppArmor DFA OOB greater than supported\n");
         *                goto fail;
         *        }
         * }
         */
        dfa->max_oob = 1;

        data += hsize;
        size -= hsize;

        while (size > 0) {
                table = unpack_table(data, size);
                if (IS_ERR(table)) {
                        error = PTR_ERR(table);
                        table = NULL;
                        goto fail;
                }

                switch (table->td_id) {
                case YYTD_ID_ACCEPT:
                        if (!(table->td_flags & ACCEPT1_FLAGS(flags)))
                                goto fail;
                        break;
                case YYTD_ID_ACCEPT2:
                        if (!(table->td_flags & ACCEPT2_FLAGS(flags)))
                                goto fail;
                        break;
                case YYTD_ID_BASE:
                        if (table->td_flags != YYTD_DATA32)
                                goto fail;
                        break;
                case YYTD_ID_DEF:
                case YYTD_ID_NXT:
                case YYTD_ID_CHK:
                        if (!(table->td_flags == YYTD_DATA16 ||
                              table->td_flags == YYTD_DATA32)) {
                                goto fail;
                        }
                        break;
                case YYTD_ID_EC:
                        if (table->td_flags != YYTD_DATA8)
                                goto fail;
                        break;
                default:
                        goto fail;
                }
                /* check for duplicate table entry */
                if (dfa->tables[table->td_id])
                        goto fail;
                dfa->tables[table->td_id] = table;
                data += table_size(table->td_lolen, table->td_flags);
                size -= table_size(table->td_lolen, table->td_flags);

                /*
                 * this remapping has to be done after incrementing data above
                 * for now straight remap, later have dfa support both
                 */
                switch (table->td_id) {
                case YYTD_ID_DEF:
                case YYTD_ID_NXT:
                case YYTD_ID_CHK:
                        if (table->td_flags == YYTD_DATA16) {
                                table = remap_data16_to_data32(table);
                                if (!table)
                                        goto fail;
                        }
                        dfa->tables[table->td_id] = table;
                        break;
                }
                table = NULL;
        }
        error = verify_table_headers(dfa->tables, flags);
        if (error)
                goto fail;

        if (flags & DFA_FLAG_VERIFY_STATES) {
                error = verify_dfa(dfa);
                if (error)
                        goto fail;
        }

        return dfa;

fail:
        kvfree(table);
        dfa_free(dfa);
        return ERR_PTR(error);
}

#define match_char(state, def, base, next, check, C)        \
do {                                                        \
        u32 b = (base)[(state)];                        \
        unsigned int pos = base_idx(b) + (C);                \
        if ((check)[pos] != (state)) {                        \
                (state) = (def)[(state)];                \
                if (b & MATCH_FLAG_DIFF_ENCODE)                \
                        continue;                        \
                break;                                        \
        }                                                \
        (state) = (next)[pos];                                \
        break;                                                \
} while (1)

/**
 * aa_dfa_match_len - traverse @dfa to find state @str stops at
 * @dfa: the dfa to match @str against  (NOT NULL)
 * @start: the state of the dfa to start matching in
 * @str: the string of bytes to match against the dfa  (NOT NULL)
 * @len: length of the string of bytes to match
 *
 * aa_dfa_match_len will match @str against the dfa and return the state it
 * finished matching in. The final state can be used to look up the accepting
 * label, or as the start state of a continuing match.
 *
 * This function will happily match again the 0 byte and only finishes
 * when @len input is consumed.
 *
 * Returns: final state reached after input is consumed
 */
aa_state_t aa_dfa_match_len(const struct aa_dfa *dfa, aa_state_t start,
                            const char *str, int len)
{
        u32 *def = DEFAULT_TABLE(dfa);
        u32 *base = BASE_TABLE(dfa);
        u32 *next = NEXT_TABLE(dfa);
        u32 *check = CHECK_TABLE(dfa);
        aa_state_t state = start;

        if (state == DFA_NOMATCH)
                return DFA_NOMATCH;

        /* current state is <state>, matching character *str */
        if (dfa->tables[YYTD_ID_EC]) {
                /* Equivalence class table defined */
                u8 *equiv = EQUIV_TABLE(dfa);
                for (; len; len--) {
                        u8 c = equiv[(u8) *str];

                        match_char(state, def, base, next, check, c);
                        str++;
                }
        } else {
                /* default is direct to next state */
                for (; len; len--) {
                        match_char(state, def, base, next, check, (u8) *str);
                        str++;
                }
        }

        return state;
}

/**
 * aa_dfa_match - traverse @dfa to find state @str stops at
 * @dfa: the dfa to match @str against  (NOT NULL)
 * @start: the state of the dfa to start matching in
 * @str: the null terminated string of bytes to match against the dfa (NOT NULL)
 *
 * aa_dfa_match will match @str against the dfa and return the state it
 * finished matching in. The final state can be used to look up the accepting
 * label, or as the start state of a continuing match.
 *
 * Returns: final state reached after input is consumed
 */
aa_state_t aa_dfa_match(const struct aa_dfa *dfa, aa_state_t start,
                        const char *str)
{
        u32 *def = DEFAULT_TABLE(dfa);
        u32 *base = BASE_TABLE(dfa);
        u32 *next = NEXT_TABLE(dfa);
        u32 *check = CHECK_TABLE(dfa);
        aa_state_t state = start;

        if (state == DFA_NOMATCH)
                return DFA_NOMATCH;

        /* current state is <state>, matching character *str */
        if (dfa->tables[YYTD_ID_EC]) {
                /* Equivalence class table defined */
                u8 *equiv = EQUIV_TABLE(dfa);
                /* default is direct to next state */
                while (*str) {
                        u8 c = equiv[(u8) *str];

                        match_char(state, def, base, next, check, c);
                        str++;
                }
        } else {
                /* default is direct to next state */
                while (*str) {
                        match_char(state, def, base, next, check, (u8) *str);
                        str++;
                }
        }

        return state;
}

/**
 * aa_dfa_next - step one character to the next state in the dfa
 * @dfa: the dfa to traverse (NOT NULL)
 * @state: the state to start in
 * @c: the input character to transition on
 *
 * aa_dfa_match will step through the dfa by one input character @c
 *
 * Returns: state reach after input @c
 */
aa_state_t aa_dfa_next(const struct aa_dfa *dfa, aa_state_t state,
                       const char c)
{
        u32 *def = DEFAULT_TABLE(dfa);
        u32 *base = BASE_TABLE(dfa);
        u32 *next = NEXT_TABLE(dfa);
        u32 *check = CHECK_TABLE(dfa);

        /* current state is <state>, matching character *str */
        if (dfa->tables[YYTD_ID_EC]) {
                /* Equivalence class table defined */
                u8 *equiv = EQUIV_TABLE(dfa);
                match_char(state, def, base, next, check, equiv[(u8) c]);
        } else
                match_char(state, def, base, next, check, (u8) c);

        return state;
}

aa_state_t aa_dfa_outofband_transition(const struct aa_dfa *dfa,
                                       aa_state_t state)
{
        u32 *def = DEFAULT_TABLE(dfa);
        u32 *base = BASE_TABLE(dfa);
        u32 *next = NEXT_TABLE(dfa);
        u32 *check = CHECK_TABLE(dfa);
        u32 b = (base)[(state)];

        if (!(b & MATCH_FLAG_OOB_TRANSITION))
                return DFA_NOMATCH;

        /* No Equivalence class remapping for outofband transitions */
        match_char(state, def, base, next, check, -1);

        return state;
}

/**
 * aa_dfa_match_until - traverse @dfa until accept state or end of input
 * @dfa: the dfa to match @str against  (NOT NULL)
 * @start: the state of the dfa to start matching in
 * @str: the null terminated string of bytes to match against the dfa (NOT NULL)
 * @retpos: first character in str after match OR end of string
 *
 * aa_dfa_match will match @str against the dfa and return the state it
 * finished matching in. The final state can be used to look up the accepting
 * label, or as the start state of a continuing match.
 *
 * Returns: final state reached after input is consumed
 */
aa_state_t aa_dfa_match_until(const struct aa_dfa *dfa, aa_state_t start,
                              const char *str, const char **retpos)
{
        u32 *def = DEFAULT_TABLE(dfa);
        u32 *base = BASE_TABLE(dfa);
        u32 *next = NEXT_TABLE(dfa);
        u32 *check = CHECK_TABLE(dfa);
        u32 *accept = ACCEPT_TABLE(dfa);
        aa_state_t state = start, pos;

        if (state == DFA_NOMATCH)
                return DFA_NOMATCH;

        /* current state is <state>, matching character *str */
        if (dfa->tables[YYTD_ID_EC]) {
                /* Equivalence class table defined */
                u8 *equiv = EQUIV_TABLE(dfa);
                /* default is direct to next state */
                while (*str) {
                        pos = base_idx(base[state]) + equiv[(u8) *str++];
                        if (check[pos] == state)
                                state = next[pos];
                        else
                                state = def[state];
                        if (accept[state])
                                break;
                }
        } else {
                /* default is direct to next state */
                while (*str) {
                        pos = base_idx(base[state]) + (u8) *str++;
                        if (check[pos] == state)
                                state = next[pos];
                        else
                                state = def[state];
                        if (accept[state])
                                break;
                }
        }

        *retpos = str;
        return state;
}

/**
 * aa_dfa_matchn_until - traverse @dfa until accept or @n bytes consumed
 * @dfa: the dfa to match @str against  (NOT NULL)
 * @start: the state of the dfa to start matching in
 * @str: the string of bytes to match against the dfa  (NOT NULL)
 * @n: length of the string of bytes to match
 * @retpos: first character in str after match OR str + n
 *
 * aa_dfa_match_len will match @str against the dfa and return the state it
 * finished matching in. The final state can be used to look up the accepting
 * label, or as the start state of a continuing match.
 *
 * This function will happily match again the 0 byte and only finishes
 * when @n input is consumed.
 *
 * Returns: final state reached after input is consumed
 */
aa_state_t aa_dfa_matchn_until(const struct aa_dfa *dfa, aa_state_t start,
                               const char *str, int n, const char **retpos)
{
        u32 *def = DEFAULT_TABLE(dfa);
        u32 *base = BASE_TABLE(dfa);
        u32 *next = NEXT_TABLE(dfa);
        u32 *check = CHECK_TABLE(dfa);
        u32 *accept = ACCEPT_TABLE(dfa);
        aa_state_t state = start, pos;

        *retpos = NULL;
        if (state == DFA_NOMATCH)
                return DFA_NOMATCH;

        /* current state is <state>, matching character *str */
        if (dfa->tables[YYTD_ID_EC]) {
                /* Equivalence class table defined */
                u8 *equiv = EQUIV_TABLE(dfa);
                /* default is direct to next state */
                for (; n; n--) {
                        pos = base_idx(base[state]) + equiv[(u8) *str++];
                        if (check[pos] == state)
                                state = next[pos];
                        else
                                state = def[state];
                        if (accept[state])
                                break;
                }
        } else {
                /* default is direct to next state */
                for (; n; n--) {
                        pos = base_idx(base[state]) + (u8) *str++;
                        if (check[pos] == state)
                                state = next[pos];
                        else
                                state = def[state];
                        if (accept[state])
                                break;
                }
        }

        *retpos = str;
        return state;
}

#define inc_wb_pos(wb)                                                        \
do {                                                                        \
        BUILD_BUG_ON_NOT_POWER_OF_2(WB_HISTORY_SIZE);                        \
        wb->pos = (wb->pos + 1) & (WB_HISTORY_SIZE - 1);                \
        wb->len = (wb->len + 1) > WB_HISTORY_SIZE ? WB_HISTORY_SIZE :        \
                                wb->len + 1;                                \
} while (0)

/* For DFAs that don't support extended tagging of states */
/* adjust is only set if is_loop returns true */
static bool is_loop(struct match_workbuf *wb, aa_state_t state,
                    unsigned int *adjust)
{
        int pos = wb->pos;
        int i;

        if (wb->history[pos] < state)
                return false;

        for (i = 0; i < wb->len; i++) {
                if (wb->history[pos] == state) {
                        *adjust = i;
                        return true;
                }
                /* -1 wraps to WB_HISTORY_SIZE - 1 */
                pos = (pos - 1) & (WB_HISTORY_SIZE - 1);
        }

        return false;
}

static aa_state_t leftmatch_fb(const struct aa_dfa *dfa, aa_state_t start,
                               const char *str, struct match_workbuf *wb,
                               unsigned int *count)
{
        u32 *def = DEFAULT_TABLE(dfa);
        u32 *base = BASE_TABLE(dfa);
        u32 *next = NEXT_TABLE(dfa);
        u32 *check = CHECK_TABLE(dfa);
        aa_state_t state = start, pos;

        AA_BUG(!dfa);
        AA_BUG(!str);
        AA_BUG(!wb);
        AA_BUG(!count);

        *count = 0;
        if (state == DFA_NOMATCH)
                return DFA_NOMATCH;

        /* current state is <state>, matching character *str */
        if (dfa->tables[YYTD_ID_EC]) {
                /* Equivalence class table defined */
                u8 *equiv = EQUIV_TABLE(dfa);
                /* default is direct to next state */
                while (*str) {
                        unsigned int adjust;

                        wb->history[wb->pos] = state;
                        pos = base_idx(base[state]) + equiv[(u8) *str++];
                        if (check[pos] == state)
                                state = next[pos];
                        else
                                state = def[state];
                        if (is_loop(wb, state, &adjust)) {
                                state = aa_dfa_match(dfa, state, str);
                                *count -= adjust;
                                goto out;
                        }
                        inc_wb_pos(wb);
                        (*count)++;
                }
        } else {
                /* default is direct to next state */
                while (*str) {
                        unsigned int adjust;

                        wb->history[wb->pos] = state;
                        pos = base_idx(base[state]) + (u8) *str++;
                        if (check[pos] == state)
                                state = next[pos];
                        else
                                state = def[state];
                        if (is_loop(wb, state, &adjust)) {
                                state = aa_dfa_match(dfa, state, str);
                                *count -= adjust;
                                goto out;
                        }
                        inc_wb_pos(wb);
                        (*count)++;
                }
        }

out:
        if (!state)
                *count = 0;
        return state;
}

/**
 * aa_dfa_leftmatch - traverse @dfa to find state @str stops at
 * @dfa: the dfa to match @str against  (NOT NULL)
 * @start: the state of the dfa to start matching in
 * @str: the null terminated string of bytes to match against the dfa (NOT NULL)
 * @count: current count of longest left.
 *
 * aa_dfa_match will match @str against the dfa and return the state it
 * finished matching in. The final state can be used to look up the accepting
 * label, or as the start state of a continuing match.
 *
 * Returns: final state reached after input is consumed
 */
aa_state_t aa_dfa_leftmatch(const struct aa_dfa *dfa, aa_state_t start,
                            const char *str, unsigned int *count)
{
        DEFINE_MATCH_WB(wb);

        /* TODO: match for extended state dfas */

        return leftmatch_fb(dfa, start, str, &wb, count);
}































































































































































    3 


    2 
    1 
























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
// SPDX-License-Identifier: GPL-2.0-only

#include <linux/ethtool.h>
#include <linux/sfp.h>

#include "common.h"
#include "netlink.h"

struct eeprom_req_info {
        struct ethnl_req_info        base;
        u32                        offset;
        u32                        length;
        u8                        page;
        u8                        bank;
        u8                        i2c_address;
};

struct eeprom_reply_data {
        struct ethnl_reply_data base;
        u32                        length;
        u8                        *data;
};

#define MODULE_EEPROM_REQINFO(__req_base) \
        container_of(__req_base, struct eeprom_req_info, base)

#define MODULE_EEPROM_REPDATA(__reply_base) \
        container_of(__reply_base, struct eeprom_reply_data, base)

static int fallback_set_params(struct eeprom_req_info *request,
                               struct ethtool_modinfo *modinfo,
                               struct ethtool_eeprom *eeprom)
{
        u32 offset = request->offset;
        u32 length = request->length;

        if (request->page)
                offset = request->page * ETH_MODULE_EEPROM_PAGE_LEN + offset;

        if (modinfo->type == ETH_MODULE_SFF_8472 &&
            request->i2c_address == 0x51)
                offset += ETH_MODULE_EEPROM_PAGE_LEN * 2;

        if (offset >= modinfo->eeprom_len)
                return -EINVAL;

        if (length > modinfo->eeprom_len - offset)
                return -EINVAL;

        eeprom->cmd = ETHTOOL_GMODULEEEPROM;
        eeprom->len = length;
        eeprom->offset = offset;

        return 0;
}

static int eeprom_fallback(struct eeprom_req_info *request,
                           struct eeprom_reply_data *reply)
{
        struct net_device *dev = reply->base.dev;
        struct ethtool_modinfo modinfo = {0};
        struct ethtool_eeprom eeprom = {0};
        u8 *data;
        int err;

        modinfo.cmd = ETHTOOL_GMODULEINFO;
        err = ethtool_get_module_info_call(dev, &modinfo);
        if (err < 0)
                return err;

        err = fallback_set_params(request, &modinfo, &eeprom);
        if (err < 0)
                return err;

        data = kzalloc(eeprom.len, GFP_KERNEL);
        if (!data)
                return -ENOMEM;
        err = ethtool_get_module_eeprom_call(dev, &eeprom, data);
        if (err < 0)
                goto err_out;

        reply->data = data;
        reply->length = eeprom.len;

        return 0;

err_out:
        kfree(data);
        return err;
}

static int get_module_eeprom_by_page(struct net_device *dev,
                                     struct ethtool_module_eeprom *page_data,
                                     struct netlink_ext_ack *extack)
{
        const struct ethtool_ops *ops = dev->ethtool_ops;

        if (dev->ethtool->module_fw_flash_in_progress) {
                NL_SET_ERR_MSG(extack,
                               "Module firmware flashing is in progress");
                return -EBUSY;
        }

        if (dev->sfp_bus)
                return sfp_get_module_eeprom_by_page(dev->sfp_bus, page_data, extack);

        if (ops->get_module_eeprom_by_page)
                return ops->get_module_eeprom_by_page(dev, page_data, extack);

        return -EOPNOTSUPP;
}

static int eeprom_prepare_data(const struct ethnl_req_info *req_base,
                               struct ethnl_reply_data *reply_base,
                               const struct genl_info *info)
{
        struct eeprom_reply_data *reply = MODULE_EEPROM_REPDATA(reply_base);
        struct eeprom_req_info *request = MODULE_EEPROM_REQINFO(req_base);
        struct ethtool_module_eeprom page_data = {0};
        struct net_device *dev = reply_base->dev;
        int ret;

        page_data.offset = request->offset;
        page_data.length = request->length;
        page_data.i2c_address = request->i2c_address;
        page_data.page = request->page;
        page_data.bank = request->bank;
        page_data.data = kmalloc(page_data.length, GFP_KERNEL);
        if (!page_data.data)
                return -ENOMEM;

        ret = ethnl_ops_begin(dev);
        if (ret)
                goto err_free;

        ret = get_module_eeprom_by_page(dev, &page_data, info->extack);
        if (ret < 0)
                goto err_ops;

        reply->length = ret;
        reply->data = page_data.data;

        ethnl_ops_complete(dev);
        return 0;

err_ops:
        if (ret == -EOPNOTSUPP)
                ret = eeprom_fallback(request, reply);
        ethnl_ops_complete(dev);
err_free:
        kfree(page_data.data);
        return ret;
}

static int eeprom_parse_request(struct ethnl_req_info *req_info,
                                const struct genl_info *info,
                                struct nlattr **tb,
                                struct netlink_ext_ack *extack)
{
        struct eeprom_req_info *request = MODULE_EEPROM_REQINFO(req_info);

        if (GENL_REQ_ATTR_CHECK(info, ETHTOOL_A_MODULE_EEPROM_OFFSET) ||
            GENL_REQ_ATTR_CHECK(info, ETHTOOL_A_MODULE_EEPROM_LENGTH) ||
            GENL_REQ_ATTR_CHECK(info, ETHTOOL_A_MODULE_EEPROM_PAGE) ||
            GENL_REQ_ATTR_CHECK(info, ETHTOOL_A_MODULE_EEPROM_I2C_ADDRESS))
                return -EINVAL;

        request->i2c_address = nla_get_u8(tb[ETHTOOL_A_MODULE_EEPROM_I2C_ADDRESS]);
        request->offset = nla_get_u32(tb[ETHTOOL_A_MODULE_EEPROM_OFFSET]);
        request->length = nla_get_u32(tb[ETHTOOL_A_MODULE_EEPROM_LENGTH]);

        /* The following set of conditions limit the API to only dump 1/2
         * EEPROM page without crossing low page boundary located at offset 128.
         * This means user may only request dumps of length limited to 128 from
         * either low 128 bytes or high 128 bytes.
         * For pages higher than 0 only high 128 bytes are accessible.
         */
        request->page = nla_get_u8(tb[ETHTOOL_A_MODULE_EEPROM_PAGE]);
        if (request->page && request->offset < ETH_MODULE_EEPROM_PAGE_LEN) {
                NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_MODULE_EEPROM_PAGE],
                                    "reading from lower half page is allowed for page 0 only");
                return -EINVAL;
        }

        if (request->offset < ETH_MODULE_EEPROM_PAGE_LEN &&
            request->offset + request->length > ETH_MODULE_EEPROM_PAGE_LEN) {
                NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_MODULE_EEPROM_LENGTH],
                                    "reading cross half page boundary is illegal");
                return -EINVAL;
        } else if (request->offset + request->length > ETH_MODULE_EEPROM_PAGE_LEN * 2) {
                NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_MODULE_EEPROM_LENGTH],
                                    "reading cross page boundary is illegal");
                return -EINVAL;
        }

        if (tb[ETHTOOL_A_MODULE_EEPROM_BANK])
                request->bank = nla_get_u8(tb[ETHTOOL_A_MODULE_EEPROM_BANK]);

        return 0;
}

static int eeprom_reply_size(const struct ethnl_req_info *req_base,
                             const struct ethnl_reply_data *reply_base)
{
        const struct eeprom_req_info *request = MODULE_EEPROM_REQINFO(req_base);

        return nla_total_size(sizeof(u8) * request->length); /* _EEPROM_DATA */
}

static int eeprom_fill_reply(struct sk_buff *skb,
                             const struct ethnl_req_info *req_base,
                             const struct ethnl_reply_data *reply_base)
{
        struct eeprom_reply_data *reply = MODULE_EEPROM_REPDATA(reply_base);

        return nla_put(skb, ETHTOOL_A_MODULE_EEPROM_DATA, reply->length, reply->data);
}

static void eeprom_cleanup_data(struct ethnl_reply_data *reply_base)
{
        struct eeprom_reply_data *reply = MODULE_EEPROM_REPDATA(reply_base);

        kfree(reply->data);
}

const struct ethnl_request_ops ethnl_module_eeprom_request_ops = {
        .request_cmd                = ETHTOOL_MSG_MODULE_EEPROM_GET,
        .reply_cmd                = ETHTOOL_MSG_MODULE_EEPROM_GET_REPLY,
        .hdr_attr                = ETHTOOL_A_MODULE_EEPROM_HEADER,
        .req_info_size                = sizeof(struct eeprom_req_info),
        .reply_data_size        = sizeof(struct eeprom_reply_data),

        .parse_request                = eeprom_parse_request,
        .prepare_data                = eeprom_prepare_data,
        .reply_size                = eeprom_reply_size,
        .fill_reply                = eeprom_fill_reply,
        .cleanup_data                = eeprom_cleanup_data,
};

const struct nla_policy ethnl_module_eeprom_get_policy[] = {
        [ETHTOOL_A_MODULE_EEPROM_HEADER]        = NLA_POLICY_NESTED(ethnl_header_policy),
        [ETHTOOL_A_MODULE_EEPROM_OFFSET]        =
                NLA_POLICY_MAX(NLA_U32, ETH_MODULE_EEPROM_PAGE_LEN * 2 - 1),
        [ETHTOOL_A_MODULE_EEPROM_LENGTH]        =
                NLA_POLICY_RANGE(NLA_U32, 1, ETH_MODULE_EEPROM_PAGE_LEN),
        [ETHTOOL_A_MODULE_EEPROM_PAGE]                = { .type = NLA_U8 },
        [ETHTOOL_A_MODULE_EEPROM_BANK]                = { .type = NLA_U8 },
        [ETHTOOL_A_MODULE_EEPROM_I2C_ADDRESS]        =
                NLA_POLICY_RANGE(NLA_U8, 0, ETH_MODULE_MAX_I2C_ADDRESS),
};




























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  127 












  127 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
// SPDX-License-Identifier: GPL-2.0
/*
 * linux/ipc/msg.c
 * Copyright (C) 1992 Krishna Balasubramanian
 *
 * Removed all the remaining kerneld mess
 * Catch the -EFAULT stuff properly
 * Use GFP_KERNEL for messages as in 1.2
 * Fixed up the unchecked user space derefs
 * Copyright (C) 1998 Alan Cox & Andi Kleen
 *
 * /proc/sysvipc/msg support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
 *
 * mostly rewritten, threaded and wake-one semantics added
 * MSGMAX limit removed, sysctl's added
 * (c) 1999 Manfred Spraul <manfred@colorfullife.com>
 *
 * support for audit of ipc object properties and permission changes
 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
 *
 * namespaces support
 * OpenVZ, SWsoft Inc.
 * Pavel Emelianov <xemul@openvz.org>
 */

#include <linux/capability.h>
#include <linux/msg.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/proc_fs.h>
#include <linux/list.h>
#include <linux/security.h>
#include <linux/sched/wake_q.h>
#include <linux/syscalls.h>
#include <linux/audit.h>
#include <linux/seq_file.h>
#include <linux/rwsem.h>
#include <linux/nsproxy.h>
#include <linux/ipc_namespace.h>
#include <linux/rhashtable.h>
#include <linux/percpu_counter.h>

#include <asm/current.h>
#include <linux/uaccess.h>
#include "util.h"

/* one msq_queue structure for each present queue on the system */
struct msg_queue {
        struct kern_ipc_perm q_perm;
        time64_t q_stime;                /* last msgsnd time */
        time64_t q_rtime;                /* last msgrcv time */
        time64_t q_ctime;                /* last change time */
        unsigned long q_cbytes;                /* current number of bytes on queue */
        unsigned long q_qnum;                /* number of messages in queue */
        unsigned long q_qbytes;                /* max number of bytes on queue */
        struct pid *q_lspid;                /* pid of last msgsnd */
        struct pid *q_lrpid;                /* last receive pid */

        struct list_head q_messages;
        struct list_head q_receivers;
        struct list_head q_senders;
} __randomize_layout;

/*
 * MSG_BARRIER Locking:
 *
 * Similar to the optimization used in ipc/mqueue.c, one syscall return path
 * does not acquire any locks when it sees that a message exists in
 * msg_receiver.r_msg. Therefore r_msg is set using smp_store_release()
 * and accessed using READ_ONCE()+smp_acquire__after_ctrl_dep(). In addition,
 * wake_q_add_safe() is used. See ipc/mqueue.c for more details
 */

/* one msg_receiver structure for each sleeping receiver */
struct msg_receiver {
        struct list_head        r_list;
        struct task_struct        *r_tsk;

        int                        r_mode;
        long                        r_msgtype;
        long                        r_maxsize;

        struct msg_msg                *r_msg;
};

/* one msg_sender for each sleeping sender */
struct msg_sender {
        struct list_head        list;
        struct task_struct        *tsk;
        size_t                  msgsz;
};

#define SEARCH_ANY                1
#define SEARCH_EQUAL                2
#define SEARCH_NOTEQUAL                3
#define SEARCH_LESSEQUAL        4
#define SEARCH_NUMBER                5

#define msg_ids(ns)        ((ns)->ids[IPC_MSG_IDS])

static inline struct msg_queue *msq_obtain_object(struct ipc_namespace *ns, int id)
{
        struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&msg_ids(ns), id);

        if (IS_ERR(ipcp))
                return ERR_CAST(ipcp);

        return container_of(ipcp, struct msg_queue, q_perm);
}

static inline struct msg_queue *msq_obtain_object_check(struct ipc_namespace *ns,
                                                        int id)
{
        struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&msg_ids(ns), id);

        if (IS_ERR(ipcp))
                return ERR_CAST(ipcp);

        return container_of(ipcp, struct msg_queue, q_perm);
}

static inline void msg_rmid(struct ipc_namespace *ns, struct msg_queue *s)
{
        ipc_rmid(&msg_ids(ns), &s->q_perm);
}

static void msg_rcu_free(struct rcu_head *head)
{
        struct kern_ipc_perm *p = container_of(head, struct kern_ipc_perm, rcu);
        struct msg_queue *msq = container_of(p, struct msg_queue, q_perm);

        security_msg_queue_free(&msq->q_perm);
        kfree(msq);
}

/**
 * newque - Create a new msg queue
 * @ns: namespace
 * @params: ptr to the structure that contains the key and msgflg
 *
 * Called with msg_ids.rwsem held (writer)
 */
static int newque(struct ipc_namespace *ns, struct ipc_params *params)
{
        struct msg_queue *msq;
        int retval;
        key_t key = params->key;
        int msgflg = params->flg;

        msq = kmalloc_obj(*msq, GFP_KERNEL_ACCOUNT);
        if (unlikely(!msq))
                return -ENOMEM;

        msq->q_perm.mode = msgflg & S_IRWXUGO;
        msq->q_perm.key = key;

        msq->q_perm.security = NULL;
        retval = security_msg_queue_alloc(&msq->q_perm);
        if (retval) {
                kfree(msq);
                return retval;
        }

        msq->q_stime = msq->q_rtime = 0;
        msq->q_ctime = ktime_get_real_seconds();
        msq->q_cbytes = msq->q_qnum = 0;
        msq->q_qbytes = ns->msg_ctlmnb;
        msq->q_lspid = msq->q_lrpid = NULL;
        INIT_LIST_HEAD(&msq->q_messages);
        INIT_LIST_HEAD(&msq->q_receivers);
        INIT_LIST_HEAD(&msq->q_senders);

        /* ipc_addid() locks msq upon success. */
        retval = ipc_addid(&msg_ids(ns), &msq->q_perm, ns->msg_ctlmni);
        if (retval < 0) {
                ipc_rcu_putref(&msq->q_perm, msg_rcu_free);
                return retval;
        }

        ipc_unlock_object(&msq->q_perm);
        rcu_read_unlock();

        return msq->q_perm.id;
}

static inline bool msg_fits_inqueue(struct msg_queue *msq, size_t msgsz)
{
        return msgsz + msq->q_cbytes <= msq->q_qbytes &&
                1 + msq->q_qnum <= msq->q_qbytes;
}

static inline void ss_add(struct msg_queue *msq,
                          struct msg_sender *mss, size_t msgsz)
{
        mss->tsk = current;
        mss->msgsz = msgsz;
        /*
         * No memory barrier required: we did ipc_lock_object(),
         * and the waker obtains that lock before calling wake_q_add().
         */
        __set_current_state(TASK_INTERRUPTIBLE);
        list_add_tail(&mss->list, &msq->q_senders);
}

static inline void ss_del(struct msg_sender *mss)
{
        if (mss->list.next)
                list_del(&mss->list);
}

static void ss_wakeup(struct msg_queue *msq,
                      struct wake_q_head *wake_q, bool kill)
{
        struct msg_sender *mss, *t;
        struct task_struct *stop_tsk = NULL;
        struct list_head *h = &msq->q_senders;

        list_for_each_entry_safe(mss, t, h, list) {
                if (kill)
                        mss->list.next = NULL;

                /*
                 * Stop at the first task we don't wakeup,
                 * we've already iterated the original
                 * sender queue.
                 */
                else if (stop_tsk == mss->tsk)
                        break;
                /*
                 * We are not in an EIDRM scenario here, therefore
                 * verify that we really need to wakeup the task.
                 * To maintain current semantics and wakeup order,
                 * move the sender to the tail on behalf of the
                 * blocked task.
                 */
                else if (!msg_fits_inqueue(msq, mss->msgsz)) {
                        if (!stop_tsk)
                                stop_tsk = mss->tsk;

                        list_move_tail(&mss->list, &msq->q_senders);
                        continue;
                }

                wake_q_add(wake_q, mss->tsk);
        }
}

static void expunge_all(struct msg_queue *msq, int res,
                        struct wake_q_head *wake_q)
{
        struct msg_receiver *msr, *t;

        list_for_each_entry_safe(msr, t, &msq->q_receivers, r_list) {
                struct task_struct *r_tsk;

                r_tsk = get_task_struct(msr->r_tsk);

                /* see MSG_BARRIER for purpose/pairing */
                smp_store_release(&msr->r_msg, ERR_PTR(res));
                wake_q_add_safe(wake_q, r_tsk);
        }
}

/*
 * freeque() wakes up waiters on the sender and receiver waiting queue,
 * removes the message queue from message queue ID IDR, and cleans up all the
 * messages associated with this queue.
 *
 * msg_ids.rwsem (writer) and the spinlock for this message queue are held
 * before freeque() is called. msg_ids.rwsem remains locked on exit.
 */
static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
        __releases(RCU)
        __releases(&msq->q_perm)
{
        struct msg_msg *msg, *t;
        struct msg_queue *msq = container_of(ipcp, struct msg_queue, q_perm);
        DEFINE_WAKE_Q(wake_q);

        expunge_all(msq, -EIDRM, &wake_q);
        ss_wakeup(msq, &wake_q, true);
        msg_rmid(ns, msq);
        ipc_unlock_object(&msq->q_perm);
        wake_up_q(&wake_q);
        rcu_read_unlock();

        list_for_each_entry_safe(msg, t, &msq->q_messages, m_list) {
                percpu_counter_sub_local(&ns->percpu_msg_hdrs, 1);
                free_msg(msg);
        }
        percpu_counter_sub_local(&ns->percpu_msg_bytes, msq->q_cbytes);
        ipc_update_pid(&msq->q_lspid, NULL);
        ipc_update_pid(&msq->q_lrpid, NULL);
        ipc_rcu_putref(&msq->q_perm, msg_rcu_free);
}

long ksys_msgget(key_t key, int msgflg)
{
        struct ipc_namespace *ns;
        static const struct ipc_ops msg_ops = {
                .getnew = newque,
                .associate = security_msg_queue_associate,
        };
        struct ipc_params msg_params;

        ns = current->nsproxy->ipc_ns;

        msg_params.key = key;
        msg_params.flg = msgflg;

        return ipcget(ns, &msg_ids(ns), &msg_ops, &msg_params);
}

SYSCALL_DEFINE2(msgget, key_t, key, int, msgflg)
{
        return ksys_msgget(key, msgflg);
}

static inline unsigned long
copy_msqid_to_user(void __user *buf, struct msqid64_ds *in, int version)
{
        switch (version) {
        case IPC_64:
                return copy_to_user(buf, in, sizeof(*in));
        case IPC_OLD:
        {
                struct msqid_ds out;

                memset(&out, 0, sizeof(out));

                ipc64_perm_to_ipc_perm(&in->msg_perm, &out.msg_perm);

                out.msg_stime                = in->msg_stime;
                out.msg_rtime                = in->msg_rtime;
                out.msg_ctime                = in->msg_ctime;

                if (in->msg_cbytes > USHRT_MAX)
                        out.msg_cbytes        = USHRT_MAX;
                else
                        out.msg_cbytes        = in->msg_cbytes;
                out.msg_lcbytes                = in->msg_cbytes;

                if (in->msg_qnum > USHRT_MAX)
                        out.msg_qnum        = USHRT_MAX;
                else
                        out.msg_qnum        = in->msg_qnum;

                if (in->msg_qbytes > USHRT_MAX)
                        out.msg_qbytes        = USHRT_MAX;
                else
                        out.msg_qbytes        = in->msg_qbytes;
                out.msg_lqbytes                = in->msg_qbytes;

                out.msg_lspid                = in->msg_lspid;
                out.msg_lrpid                = in->msg_lrpid;

                return copy_to_user(buf, &out, sizeof(out));
        }
        default:
                return -EINVAL;
        }
}

static inline unsigned long
copy_msqid_from_user(struct msqid64_ds *out, void __user *buf, int version)
{
        switch (version) {
        case IPC_64:
                if (copy_from_user(out, buf, sizeof(*out)))
                        return -EFAULT;
                return 0;
        case IPC_OLD:
        {
                struct msqid_ds tbuf_old;

                if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
                        return -EFAULT;

                out->msg_perm.uid        = tbuf_old.msg_perm.uid;
                out->msg_perm.gid        = tbuf_old.msg_perm.gid;
                out->msg_perm.mode        = tbuf_old.msg_perm.mode;

                if (tbuf_old.msg_qbytes == 0)
                        out->msg_qbytes        = tbuf_old.msg_lqbytes;
                else
                        out->msg_qbytes        = tbuf_old.msg_qbytes;

                return 0;
        }
        default:
                return -EINVAL;
        }
}

/*
 * This function handles some msgctl commands which require the rwsem
 * to be held in write mode.
 * NOTE: no locks must be held, the rwsem is taken inside this function.
 */
static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd,
                        struct ipc64_perm *perm, int msg_qbytes)
{
        struct kern_ipc_perm *ipcp;
        struct msg_queue *msq;
        int err;

        down_write(&msg_ids(ns).rwsem);
        rcu_read_lock();

        ipcp = ipcctl_obtain_check(ns, &msg_ids(ns), msqid, cmd,
                                      perm, msg_qbytes);
        if (IS_ERR(ipcp)) {
                err = PTR_ERR(ipcp);
                goto out_unlock1;
        }

        msq = container_of(ipcp, struct msg_queue, q_perm);

        err = security_msg_queue_msgctl(&msq->q_perm, cmd);
        if (err)
                goto out_unlock1;

        switch (cmd) {
        case IPC_RMID:
                ipc_lock_object(&msq->q_perm);
                /* freeque unlocks the ipc object and rcu */
                freeque(ns, ipcp);
                goto out_up;
        case IPC_SET:
        {
                DEFINE_WAKE_Q(wake_q);

                if (msg_qbytes > ns->msg_ctlmnb &&
                    !capable(CAP_SYS_RESOURCE)) {
                        err = -EPERM;
                        goto out_unlock1;
                }

                ipc_lock_object(&msq->q_perm);
                err = ipc_update_perm(perm, ipcp);
                if (err)
                        goto out_unlock0;

                msq->q_qbytes = msg_qbytes;

                msq->q_ctime = ktime_get_real_seconds();
                /*
                 * Sleeping receivers might be excluded by
                 * stricter permissions.
                 */
                expunge_all(msq, -EAGAIN, &wake_q);
                /*
                 * Sleeping senders might be able to send
                 * due to a larger queue size.
                 */
                ss_wakeup(msq, &wake_q, false);
                ipc_unlock_object(&msq->q_perm);
                wake_up_q(&wake_q);

                goto out_unlock1;
        }
        default:
                err = -EINVAL;
                goto out_unlock1;
        }

out_unlock0:
        ipc_unlock_object(&msq->q_perm);
out_unlock1:
        rcu_read_unlock();
out_up:
        up_write(&msg_ids(ns).rwsem);
        return err;
}

static int msgctl_info(struct ipc_namespace *ns, int msqid,
                         int cmd, struct msginfo *msginfo)
{
        int err;
        int max_idx;

        /*
         * We must not return kernel stack data.
         * due to padding, it's not enough
         * to set all member fields.
         */
        err = security_msg_queue_msgctl(NULL, cmd);
        if (err)
                return err;

        memset(msginfo, 0, sizeof(*msginfo));
        msginfo->msgmni = ns->msg_ctlmni;
        msginfo->msgmax = ns->msg_ctlmax;
        msginfo->msgmnb = ns->msg_ctlmnb;
        msginfo->msgssz = MSGSSZ;
        msginfo->msgseg = MSGSEG;
        down_read(&msg_ids(ns).rwsem);
        if (cmd == MSG_INFO)
                msginfo->msgpool = msg_ids(ns).in_use;
        max_idx = ipc_get_maxidx(&msg_ids(ns));
        up_read(&msg_ids(ns).rwsem);
        if (cmd == MSG_INFO) {
                msginfo->msgmap = min_t(int,
                                     percpu_counter_sum(&ns->percpu_msg_hdrs),
                                     INT_MAX);
                msginfo->msgtql = min_t(int,
                                     percpu_counter_sum(&ns->percpu_msg_bytes),
                                     INT_MAX);
        } else {
                msginfo->msgmap = MSGMAP;
                msginfo->msgpool = MSGPOOL;
                msginfo->msgtql = MSGTQL;
        }
        return (max_idx < 0) ? 0 : max_idx;
}

static int msgctl_stat(struct ipc_namespace *ns, int msqid,
                         int cmd, struct msqid64_ds *p)
{
        struct msg_queue *msq;
        int err;

        memset(p, 0, sizeof(*p));

        rcu_read_lock();
        if (cmd == MSG_STAT || cmd == MSG_STAT_ANY) {
                msq = msq_obtain_object(ns, msqid);
                if (IS_ERR(msq)) {
                        err = PTR_ERR(msq);
                        goto out_unlock;
                }
        } else { /* IPC_STAT */
                msq = msq_obtain_object_check(ns, msqid);
                if (IS_ERR(msq)) {
                        err = PTR_ERR(msq);
                        goto out_unlock;
                }
        }

        /* see comment for SHM_STAT_ANY */
        if (cmd == MSG_STAT_ANY)
                audit_ipc_obj(&msq->q_perm);
        else {
                err = -EACCES;
                if (ipcperms(ns, &msq->q_perm, S_IRUGO))
                        goto out_unlock;
        }

        err = security_msg_queue_msgctl(&msq->q_perm, cmd);
        if (err)
                goto out_unlock;

        ipc_lock_object(&msq->q_perm);

        if (!ipc_valid_object(&msq->q_perm)) {
                ipc_unlock_object(&msq->q_perm);
                err = -EIDRM;
                goto out_unlock;
        }

        kernel_to_ipc64_perm(&msq->q_perm, &p->msg_perm);
        p->msg_stime  = msq->q_stime;
        p->msg_rtime  = msq->q_rtime;
        p->msg_ctime  = msq->q_ctime;
#ifndef CONFIG_64BIT
        p->msg_stime_high = msq->q_stime >> 32;
        p->msg_rtime_high = msq->q_rtime >> 32;
        p->msg_ctime_high = msq->q_ctime >> 32;
#endif
        p->msg_cbytes = msq->q_cbytes;
        p->msg_qnum   = msq->q_qnum;
        p->msg_qbytes = msq->q_qbytes;
        p->msg_lspid  = pid_vnr(msq->q_lspid);
        p->msg_lrpid  = pid_vnr(msq->q_lrpid);

        if (cmd == IPC_STAT) {
                /*
                 * As defined in SUS:
                 * Return 0 on success
                 */
                err = 0;
        } else {
                /*
                 * MSG_STAT and MSG_STAT_ANY (both Linux specific)
                 * Return the full id, including the sequence number
                 */
                err = msq->q_perm.id;
        }

        ipc_unlock_object(&msq->q_perm);
out_unlock:
        rcu_read_unlock();
        return err;
}

static long ksys_msgctl(int msqid, int cmd, struct msqid_ds __user *buf, int version)
{
        struct ipc_namespace *ns;
        struct msqid64_ds msqid64;
        int err;

        if (msqid < 0 || cmd < 0)
                return -EINVAL;

        ns = current->nsproxy->ipc_ns;

        switch (cmd) {
        case IPC_INFO:
        case MSG_INFO: {
                struct msginfo msginfo;
                err = msgctl_info(ns, msqid, cmd, &msginfo);
                if (err < 0)
                        return err;
                if (copy_to_user(buf, &msginfo, sizeof(struct msginfo)))
                        err = -EFAULT;
                return err;
        }
        case MSG_STAT:        /* msqid is an index rather than a msg queue id */
        case MSG_STAT_ANY:
        case IPC_STAT:
                err = msgctl_stat(ns, msqid, cmd, &msqid64);
                if (err < 0)
                        return err;
                if (copy_msqid_to_user(buf, &msqid64, version))
                        err = -EFAULT;
                return err;
        case IPC_SET:
                if (copy_msqid_from_user(&msqid64, buf, version))
                        return -EFAULT;
                return msgctl_down(ns, msqid, cmd, &msqid64.msg_perm,
                                   msqid64.msg_qbytes);
        case IPC_RMID:
                return msgctl_down(ns, msqid, cmd, NULL, 0);
        default:
                return  -EINVAL;
        }
}

SYSCALL_DEFINE3(msgctl, int, msqid, int, cmd, struct msqid_ds __user *, buf)
{
        return ksys_msgctl(msqid, cmd, buf, IPC_64);
}

#ifdef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
long ksys_old_msgctl(int msqid, int cmd, struct msqid_ds __user *buf)
{
        int version = ipc_parse_version(&cmd);

        return ksys_msgctl(msqid, cmd, buf, version);
}

SYSCALL_DEFINE3(old_msgctl, int, msqid, int, cmd, struct msqid_ds __user *, buf)
{
        return ksys_old_msgctl(msqid, cmd, buf);
}
#endif

#ifdef CONFIG_COMPAT

struct compat_msqid_ds {
        struct compat_ipc_perm msg_perm;
        compat_uptr_t msg_first;
        compat_uptr_t msg_last;
        old_time32_t msg_stime;
        old_time32_t msg_rtime;
        old_time32_t msg_ctime;
        compat_ulong_t msg_lcbytes;
        compat_ulong_t msg_lqbytes;
        unsigned short msg_cbytes;
        unsigned short msg_qnum;
        unsigned short msg_qbytes;
        compat_ipc_pid_t msg_lspid;
        compat_ipc_pid_t msg_lrpid;
};

static int copy_compat_msqid_from_user(struct msqid64_ds *out, void __user *buf,
                                        int version)
{
        memset(out, 0, sizeof(*out));
        if (version == IPC_64) {
                struct compat_msqid64_ds __user *p = buf;
                if (get_compat_ipc64_perm(&out->msg_perm, &p->msg_perm))
                        return -EFAULT;
                if (get_user(out->msg_qbytes, &p->msg_qbytes))
                        return -EFAULT;
        } else {
                struct compat_msqid_ds __user *p = buf;
                if (get_compat_ipc_perm(&out->msg_perm, &p->msg_perm))
                        return -EFAULT;
                if (get_user(out->msg_qbytes, &p->msg_qbytes))
                        return -EFAULT;
        }
        return 0;
}

static int copy_compat_msqid_to_user(void __user *buf, struct msqid64_ds *in,
                                        int version)
{
        if (version == IPC_64) {
                struct compat_msqid64_ds v;
                memset(&v, 0, sizeof(v));
                to_compat_ipc64_perm(&v.msg_perm, &in->msg_perm);
                v.msg_stime         = lower_32_bits(in->msg_stime);
                v.msg_stime_high = upper_32_bits(in->msg_stime);
                v.msg_rtime         = lower_32_bits(in->msg_rtime);
                v.msg_rtime_high = upper_32_bits(in->msg_rtime);
                v.msg_ctime         = lower_32_bits(in->msg_ctime);
                v.msg_ctime_high = upper_32_bits(in->msg_ctime);
                v.msg_cbytes = in->msg_cbytes;
                v.msg_qnum = in->msg_qnum;
                v.msg_qbytes = in->msg_qbytes;
                v.msg_lspid = in->msg_lspid;
                v.msg_lrpid = in->msg_lrpid;
                return copy_to_user(buf, &v, sizeof(v));
        } else {
                struct compat_msqid_ds v;
                memset(&v, 0, sizeof(v));
                to_compat_ipc_perm(&v.msg_perm, &in->msg_perm);
                v.msg_stime = in->msg_stime;
                v.msg_rtime = in->msg_rtime;
                v.msg_ctime = in->msg_ctime;
                v.msg_cbytes = in->msg_cbytes;
                v.msg_qnum = in->msg_qnum;
                v.msg_qbytes = in->msg_qbytes;
                v.msg_lspid = in->msg_lspid;
                v.msg_lrpid = in->msg_lrpid;
                return copy_to_user(buf, &v, sizeof(v));
        }
}

static long compat_ksys_msgctl(int msqid, int cmd, void __user *uptr, int version)
{
        struct ipc_namespace *ns;
        int err;
        struct msqid64_ds msqid64;

        ns = current->nsproxy->ipc_ns;

        if (msqid < 0 || cmd < 0)
                return -EINVAL;

        switch (cmd & (~IPC_64)) {
        case IPC_INFO:
        case MSG_INFO: {
                struct msginfo msginfo;
                err = msgctl_info(ns, msqid, cmd, &msginfo);
                if (err < 0)
                        return err;
                if (copy_to_user(uptr, &msginfo, sizeof(struct msginfo)))
                        err = -EFAULT;
                return err;
        }
        case IPC_STAT:
        case MSG_STAT:
        case MSG_STAT_ANY:
                err = msgctl_stat(ns, msqid, cmd, &msqid64);
                if (err < 0)
                        return err;
                if (copy_compat_msqid_to_user(uptr, &msqid64, version))
                        err = -EFAULT;
                return err;
        case IPC_SET:
                if (copy_compat_msqid_from_user(&msqid64, uptr, version))
                        return -EFAULT;
                return msgctl_down(ns, msqid, cmd, &msqid64.msg_perm, msqid64.msg_qbytes);
        case IPC_RMID:
                return msgctl_down(ns, msqid, cmd, NULL, 0);
        default:
                return -EINVAL;
        }
}

COMPAT_SYSCALL_DEFINE3(msgctl, int, msqid, int, cmd, void __user *, uptr)
{
        return compat_ksys_msgctl(msqid, cmd, uptr, IPC_64);
}

#ifdef CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION
long compat_ksys_old_msgctl(int msqid, int cmd, void __user *uptr)
{
        int version = compat_ipc_parse_version(&cmd);

        return compat_ksys_msgctl(msqid, cmd, uptr, version);
}

COMPAT_SYSCALL_DEFINE3(old_msgctl, int, msqid, int, cmd, void __user *, uptr)
{
        return compat_ksys_old_msgctl(msqid, cmd, uptr);
}
#endif
#endif

static int testmsg(struct msg_msg *msg, long type, int mode)
{
        switch (mode) {
        case SEARCH_ANY:
        case SEARCH_NUMBER:
                return 1;
        case SEARCH_LESSEQUAL:
                if (msg->m_type <= type)
                        return 1;
                break;
        case SEARCH_EQUAL:
                if (msg->m_type == type)
                        return 1;
                break;
        case SEARCH_NOTEQUAL:
                if (msg->m_type != type)
                        return 1;
                break;
        }
        return 0;
}

static inline int pipelined_send(struct msg_queue *msq, struct msg_msg *msg,
                                 struct wake_q_head *wake_q)
{
        struct msg_receiver *msr, *t;

        list_for_each_entry_safe(msr, t, &msq->q_receivers, r_list) {
                if (testmsg(msg, msr->r_msgtype, msr->r_mode) &&
                    !security_msg_queue_msgrcv(&msq->q_perm, msg, msr->r_tsk,
                                               msr->r_msgtype, msr->r_mode)) {

                        list_del(&msr->r_list);
                        if (msr->r_maxsize < msg->m_ts) {
                                wake_q_add(wake_q, msr->r_tsk);

                                /* See expunge_all regarding memory barrier */
                                smp_store_release(&msr->r_msg, ERR_PTR(-E2BIG));
                        } else {
                                ipc_update_pid(&msq->q_lrpid, task_pid(msr->r_tsk));
                                msq->q_rtime = ktime_get_real_seconds();

                                wake_q_add(wake_q, msr->r_tsk);

                                /* See expunge_all regarding memory barrier */
                                smp_store_release(&msr->r_msg, msg);
                                return 1;
                        }
                }
        }

        return 0;
}

static long do_msgsnd(int msqid, long mtype, void __user *mtext,
                size_t msgsz, int msgflg)
{
        struct msg_queue *msq;
        struct msg_msg *msg;
        int err;
        struct ipc_namespace *ns;
        DEFINE_WAKE_Q(wake_q);

        ns = current->nsproxy->ipc_ns;

        if (msgsz > ns->msg_ctlmax || (long) msgsz < 0 || msqid < 0)
                return -EINVAL;
        if (mtype < 1)
                return -EINVAL;

        msg = load_msg(mtext, msgsz);
        if (IS_ERR(msg))
                return PTR_ERR(msg);

        msg->m_type = mtype;
        msg->m_ts = msgsz;

        rcu_read_lock();
        msq = msq_obtain_object_check(ns, msqid);
        if (IS_ERR(msq)) {
                err = PTR_ERR(msq);
                goto out_unlock1;
        }

        ipc_lock_object(&msq->q_perm);

        for (;;) {
                struct msg_sender s;

                err = -EACCES;
                if (ipcperms(ns, &msq->q_perm, S_IWUGO))
                        goto out_unlock0;

                /* raced with RMID? */
                if (!ipc_valid_object(&msq->q_perm)) {
                        err = -EIDRM;
                        goto out_unlock0;
                }

                err = security_msg_queue_msgsnd(&msq->q_perm, msg, msgflg);
                if (err)
                        goto out_unlock0;

                if (msg_fits_inqueue(msq, msgsz))
                        break;

                /* queue full, wait: */
                if (msgflg & IPC_NOWAIT) {
                        err = -EAGAIN;
                        goto out_unlock0;
                }

                /* enqueue the sender and prepare to block */
                ss_add(msq, &s, msgsz);

                if (!ipc_rcu_getref(&msq->q_perm)) {
                        err = -EIDRM;
                        goto out_unlock0;
                }

                ipc_unlock_object(&msq->q_perm);
                rcu_read_unlock();
                schedule();

                rcu_read_lock();
                ipc_lock_object(&msq->q_perm);

                ipc_rcu_putref(&msq->q_perm, msg_rcu_free);
                /* raced with RMID? */
                if (!ipc_valid_object(&msq->q_perm)) {
                        err = -EIDRM;
                        goto out_unlock0;
                }
                ss_del(&s);

                if (signal_pending(current)) {
                        err = -ERESTARTNOHAND;
                        goto out_unlock0;
                }

        }

        ipc_update_pid(&msq->q_lspid, task_tgid(current));
        msq->q_stime = ktime_get_real_seconds();

        if (!pipelined_send(msq, msg, &wake_q)) {
                /* no one is waiting for this message, enqueue it */
                list_add_tail(&msg->m_list, &msq->q_messages);
                msq->q_cbytes += msgsz;
                msq->q_qnum++;
                percpu_counter_add_local(&ns->percpu_msg_bytes, msgsz);
                percpu_counter_add_local(&ns->percpu_msg_hdrs, 1);
        }

        err = 0;
        msg = NULL;

out_unlock0:
        ipc_unlock_object(&msq->q_perm);
        wake_up_q(&wake_q);
out_unlock1:
        rcu_read_unlock();
        if (msg != NULL)
                free_msg(msg);
        return err;
}

long ksys_msgsnd(int msqid, struct msgbuf __user *msgp, size_t msgsz,
                 int msgflg)
{
        long mtype;

        if (get_user(mtype, &msgp->mtype))
                return -EFAULT;
        return do_msgsnd(msqid, mtype, msgp->mtext, msgsz, msgflg);
}

SYSCALL_DEFINE4(msgsnd, int, msqid, struct msgbuf __user *, msgp, size_t, msgsz,
                int, msgflg)
{
        return ksys_msgsnd(msqid, msgp, msgsz, msgflg);
}

#ifdef CONFIG_COMPAT

struct compat_msgbuf {
        compat_long_t mtype;
        char mtext[];
};

long compat_ksys_msgsnd(int msqid, compat_uptr_t msgp,
                       compat_ssize_t msgsz, int msgflg)
{
        struct compat_msgbuf __user *up = compat_ptr(msgp);
        compat_long_t mtype;

        if (get_user(mtype, &up->mtype))
                return -EFAULT;
        return do_msgsnd(msqid, mtype, up->mtext, (ssize_t)msgsz, msgflg);
}

COMPAT_SYSCALL_DEFINE4(msgsnd, int, msqid, compat_uptr_t, msgp,
                       compat_ssize_t, msgsz, int, msgflg)
{
        return compat_ksys_msgsnd(msqid, msgp, msgsz, msgflg);
}
#endif

static inline int convert_mode(long *msgtyp, int msgflg)
{
        if (msgflg & MSG_COPY)
                return SEARCH_NUMBER;
        /*
         *  find message of correct type.
         *  msgtyp = 0 => get first.
         *  msgtyp > 0 => get first message of matching type.
         *  msgtyp < 0 => get message with least type must be < abs(msgtype).
         */
        if (*msgtyp == 0)
                return SEARCH_ANY;
        if (*msgtyp < 0) {
                if (*msgtyp == LONG_MIN) /* -LONG_MIN is undefined */
                        *msgtyp = LONG_MAX;
                else
                        *msgtyp = -*msgtyp;
                return SEARCH_LESSEQUAL;
        }
        if (msgflg & MSG_EXCEPT)
                return SEARCH_NOTEQUAL;
        return SEARCH_EQUAL;
}

static long do_msg_fill(void __user *dest, struct msg_msg *msg, size_t bufsz)
{
        struct msgbuf __user *msgp = dest;
        size_t msgsz;

        if (put_user(msg->m_type, &msgp->mtype))
                return -EFAULT;

        msgsz = (bufsz > msg->m_ts) ? msg->m_ts : bufsz;
        if (store_msg(msgp->mtext, msg, msgsz))
                return -EFAULT;
        return msgsz;
}

#ifdef CONFIG_CHECKPOINT_RESTORE
/*
 * This function creates new kernel message structure, large enough to store
 * bufsz message bytes.
 */
static inline struct msg_msg *prepare_copy(void __user *buf, size_t bufsz)
{
        struct msg_msg *copy;

        /*
         * Create dummy message to copy real message to.
         */
        copy = load_msg(buf, bufsz);
        if (!IS_ERR(copy))
                copy->m_ts = bufsz;
        return copy;
}

static inline void free_copy(struct msg_msg *copy)
{
        if (copy)
                free_msg(copy);
}
#else
static inline struct msg_msg *prepare_copy(void __user *buf, size_t bufsz)
{
        return ERR_PTR(-ENOSYS);
}

static inline void free_copy(struct msg_msg *copy)
{
}
#endif

static struct msg_msg *find_msg(struct msg_queue *msq, long *msgtyp, int mode)
{
        struct msg_msg *msg, *found = NULL;
        long count = 0;

        list_for_each_entry(msg, &msq->q_messages, m_list) {
                if (testmsg(msg, *msgtyp, mode) &&
                    !security_msg_queue_msgrcv(&msq->q_perm, msg, current,
                                               *msgtyp, mode)) {
                        if (mode == SEARCH_LESSEQUAL && msg->m_type != 1) {
                                *msgtyp = msg->m_type - 1;
                                found = msg;
                        } else if (mode == SEARCH_NUMBER) {
                                if (*msgtyp == count)
                                        return msg;
                        } else
                                return msg;
                        count++;
                }
        }

        return found ?: ERR_PTR(-EAGAIN);
}

static long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, int msgflg,
               long (*msg_handler)(void __user *, struct msg_msg *, size_t))
{
        int mode;
        struct msg_queue *msq;
        struct ipc_namespace *ns;
        struct msg_msg *msg, *copy = NULL;
        DEFINE_WAKE_Q(wake_q);

        ns = current->nsproxy->ipc_ns;

        if (msqid < 0 || (long) bufsz < 0)
                return -EINVAL;

        if (msgflg & MSG_COPY) {
                if ((msgflg & MSG_EXCEPT) || !(msgflg & IPC_NOWAIT))
                        return -EINVAL;
                copy = prepare_copy(buf, min_t(size_t, bufsz, ns->msg_ctlmax));
                if (IS_ERR(copy))
                        return PTR_ERR(copy);
        }
        mode = convert_mode(&msgtyp, msgflg);

        rcu_read_lock();
        msq = msq_obtain_object_check(ns, msqid);
        if (IS_ERR(msq)) {
                rcu_read_unlock();
                free_copy(copy);
                return PTR_ERR(msq);
        }

        for (;;) {
                struct msg_receiver msr_d;

                msg = ERR_PTR(-EACCES);
                if (ipcperms(ns, &msq->q_perm, S_IRUGO))
                        goto out_unlock1;

                ipc_lock_object(&msq->q_perm);

                /* raced with RMID? */
                if (!ipc_valid_object(&msq->q_perm)) {
                        msg = ERR_PTR(-EIDRM);
                        goto out_unlock0;
                }

                msg = find_msg(msq, &msgtyp, mode);
                if (!IS_ERR(msg)) {
                        /*
                         * Found a suitable message.
                         * Unlink it from the queue.
                         */
                        if ((bufsz < msg->m_ts) && !(msgflg & MSG_NOERROR)) {
                                msg = ERR_PTR(-E2BIG);
                                goto out_unlock0;
                        }
                        /*
                         * If we are copying, then do not unlink message and do
                         * not update queue parameters.
                         */
                        if (msgflg & MSG_COPY) {
                                msg = copy_msg(msg, copy);
                                goto out_unlock0;
                        }

                        list_del(&msg->m_list);
                        msq->q_qnum--;
                        msq->q_rtime = ktime_get_real_seconds();
                        ipc_update_pid(&msq->q_lrpid, task_tgid(current));
                        msq->q_cbytes -= msg->m_ts;
                        percpu_counter_sub_local(&ns->percpu_msg_bytes, msg->m_ts);
                        percpu_counter_sub_local(&ns->percpu_msg_hdrs, 1);
                        ss_wakeup(msq, &wake_q, false);

                        goto out_unlock0;
                }

                /* No message waiting. Wait for a message */
                if (msgflg & IPC_NOWAIT) {
                        msg = ERR_PTR(-ENOMSG);
                        goto out_unlock0;
                }

                list_add_tail(&msr_d.r_list, &msq->q_receivers);
                msr_d.r_tsk = current;
                msr_d.r_msgtype = msgtyp;
                msr_d.r_mode = mode;
                if (msgflg & MSG_NOERROR)
                        msr_d.r_maxsize = INT_MAX;
                else
                        msr_d.r_maxsize = bufsz;

                /* memory barrier not require due to ipc_lock_object() */
                WRITE_ONCE(msr_d.r_msg, ERR_PTR(-EAGAIN));

                /* memory barrier not required, we own ipc_lock_object() */
                __set_current_state(TASK_INTERRUPTIBLE);

                ipc_unlock_object(&msq->q_perm);
                rcu_read_unlock();
                schedule();

                /*
                 * Lockless receive, part 1:
                 * We don't hold a reference to the queue and getting a
                 * reference would defeat the idea of a lockless operation,
                 * thus the code relies on rcu to guarantee the existence of
                 * msq:
                 * Prior to destruction, expunge_all(-EIRDM) changes r_msg.
                 * Thus if r_msg is -EAGAIN, then the queue not yet destroyed.
                 */
                rcu_read_lock();

                /*
                 * Lockless receive, part 2:
                 * The work in pipelined_send() and expunge_all():
                 * - Set pointer to message
                 * - Queue the receiver task for later wakeup
                 * - Wake up the process after the lock is dropped.
                 *
                 * Should the process wake up before this wakeup (due to a
                 * signal) it will either see the message and continue ...
                 */
                msg = READ_ONCE(msr_d.r_msg);
                if (msg != ERR_PTR(-EAGAIN)) {
                        /* see MSG_BARRIER for purpose/pairing */
                        smp_acquire__after_ctrl_dep();

                        goto out_unlock1;
                }

                 /*
                  * ... or see -EAGAIN, acquire the lock to check the message
                  * again.
                  */
                ipc_lock_object(&msq->q_perm);

                msg = READ_ONCE(msr_d.r_msg);
                if (msg != ERR_PTR(-EAGAIN))
                        goto out_unlock0;

                list_del(&msr_d.r_list);
                if (signal_pending(current)) {
                        msg = ERR_PTR(-ERESTARTNOHAND);
                        goto out_unlock0;
                }

                ipc_unlock_object(&msq->q_perm);
        }

out_unlock0:
        ipc_unlock_object(&msq->q_perm);
        wake_up_q(&wake_q);
out_unlock1:
        rcu_read_unlock();
        if (IS_ERR(msg)) {
                free_copy(copy);
                return PTR_ERR(msg);
        }

        bufsz = msg_handler(buf, msg, bufsz);
        free_msg(msg);

        return bufsz;
}

long ksys_msgrcv(int msqid, struct msgbuf __user *msgp, size_t msgsz,
                 long msgtyp, int msgflg)
{
        return do_msgrcv(msqid, msgp, msgsz, msgtyp, msgflg, do_msg_fill);
}

SYSCALL_DEFINE5(msgrcv, int, msqid, struct msgbuf __user *, msgp, size_t, msgsz,
                long, msgtyp, int, msgflg)
{
        return ksys_msgrcv(msqid, msgp, msgsz, msgtyp, msgflg);
}

#ifdef CONFIG_COMPAT
static long compat_do_msg_fill(void __user *dest, struct msg_msg *msg, size_t bufsz)
{
        struct compat_msgbuf __user *msgp = dest;
        size_t msgsz;

        if (put_user(msg->m_type, &msgp->mtype))
                return -EFAULT;

        msgsz = (bufsz > msg->m_ts) ? msg->m_ts : bufsz;
        if (store_msg(msgp->mtext, msg, msgsz))
                return -EFAULT;
        return msgsz;
}

long compat_ksys_msgrcv(int msqid, compat_uptr_t msgp, compat_ssize_t msgsz,
                        compat_long_t msgtyp, int msgflg)
{
        return do_msgrcv(msqid, compat_ptr(msgp), (ssize_t)msgsz, (long)msgtyp,
                         msgflg, compat_do_msg_fill);
}

COMPAT_SYSCALL_DEFINE5(msgrcv, int, msqid, compat_uptr_t, msgp,
                       compat_ssize_t, msgsz, compat_long_t, msgtyp,
                       int, msgflg)
{
        return compat_ksys_msgrcv(msqid, msgp, msgsz, msgtyp, msgflg);
}
#endif

int msg_init_ns(struct ipc_namespace *ns)
{
        int ret;

        ns->msg_ctlmax = MSGMAX;
        ns->msg_ctlmnb = MSGMNB;
        ns->msg_ctlmni = MSGMNI;

        ret = percpu_counter_init(&ns->percpu_msg_bytes, 0, GFP_KERNEL);
        if (ret)
                goto fail_msg_bytes;
        ret = percpu_counter_init(&ns->percpu_msg_hdrs, 0, GFP_KERNEL);
        if (ret)
                goto fail_msg_hdrs;
        ipc_init_ids(&ns->ids[IPC_MSG_IDS]);
        return 0;

fail_msg_hdrs:
        percpu_counter_destroy(&ns->percpu_msg_bytes);
fail_msg_bytes:
        return ret;
}

#ifdef CONFIG_IPC_NS
void msg_exit_ns(struct ipc_namespace *ns)
{
        free_ipcs(ns, &msg_ids(ns), freeque);
        idr_destroy(&ns->ids[IPC_MSG_IDS].ipcs_idr);
        rhashtable_destroy(&ns->ids[IPC_MSG_IDS].key_ht);
        percpu_counter_destroy(&ns->percpu_msg_bytes);
        percpu_counter_destroy(&ns->percpu_msg_hdrs);
}
#endif

#ifdef CONFIG_PROC_FS
static int sysvipc_msg_proc_show(struct seq_file *s, void *it)
{
        struct pid_namespace *pid_ns = ipc_seq_pid_ns(s);
        struct user_namespace *user_ns = seq_user_ns(s);
        struct kern_ipc_perm *ipcp = it;
        struct msg_queue *msq = container_of(ipcp, struct msg_queue, q_perm);

        seq_printf(s,
                   "%10d %10d  %4o  %10lu %10lu %5u %5u %5u %5u %5u %5u %10llu %10llu %10llu\n",
                   msq->q_perm.key,
                   msq->q_perm.id,
                   msq->q_perm.mode,
                   msq->q_cbytes,
                   msq->q_qnum,
                   pid_nr_ns(msq->q_lspid, pid_ns),
                   pid_nr_ns(msq->q_lrpid, pid_ns),
                   from_kuid_munged(user_ns, msq->q_perm.uid),
                   from_kgid_munged(user_ns, msq->q_perm.gid),
                   from_kuid_munged(user_ns, msq->q_perm.cuid),
                   from_kgid_munged(user_ns, msq->q_perm.cgid),
                   msq->q_stime,
                   msq->q_rtime,
                   msq->q_ctime);

        return 0;
}
#endif

void __init msg_init(void)
{
        msg_init_ns(&init_ipc_ns);

        ipc_init_proc_interface("sysvipc/msg",
                                "       key      msqid perms      cbytes       qnum lspid lrpid   uid   gid  cuid  cgid      stime      rtime      ctime\n",
                                IPC_MSG_IDS, sysvipc_msg_proc_show);
}






















































    3 








  566 
  566 

  566 




   10 












    9 





    7 
    2 











    4 


    4 










   15 











    3 


    3 





   10 




   10 

   10 



   10 


    4 
    6 






    6 

    6 

    6 

    6 

    8 
    4 
    8 



   10 
   10 




   22 















   14 



    2 






    7 



   21 








   27 



   27 



    2 









    1 




    4 



    1 


   21 






   22 




    3 





    2 






    1 




    1 







   11 






   11 









   11 






   11 







   11 







   11 











    1 








    1 








    1 






    1 



































   10 













   10 


   10 








    1 









    1 















    1 



























    1 






















   28 







   12 










    3 



   16 



    1 




   15 
    6 


    9 








    7 







    3 





    3 













    3 





    3 







   13 



    1 




   12 







    3 





    3 






   10 





   10 











    1 








    1 














    1 















    1 













    1 



    1 

    7 





    1 



    9 



    9 























    3 




    2 



    1 







    3 











    6 































   11 





  530 


  530 



  530 
    4 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2017-2018 Netronome Systems, Inc.
 */

#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/bug.h>
#include <linux/kdev_t.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/netdevice.h>
#include <linux/printk.h>
#include <linux/proc_ns.h>
#include <linux/rhashtable.h>
#include <linux/rtnetlink.h>
#include <linux/rwsem.h>
#include <net/netdev_lock.h>
#include <net/xdp.h>

/* Protects offdevs, members of bpf_offload_netdev and offload members
 * of all progs.
 * RTNL lock cannot be taken when holding this lock.
 */
static DECLARE_RWSEM(bpf_devs_lock);

struct bpf_offload_dev {
        const struct bpf_prog_offload_ops *ops;
        struct list_head netdevs;
        void *priv;
};

struct bpf_offload_netdev {
        struct rhash_head l;
        struct net_device *netdev;
        struct bpf_offload_dev *offdev; /* NULL when bound-only */
        struct list_head progs;
        struct list_head maps;
        struct list_head offdev_netdevs;
};

static const struct rhashtable_params offdevs_params = {
        .nelem_hint                = 4,
        .key_len                = sizeof(struct net_device *),
        .key_offset                = offsetof(struct bpf_offload_netdev, netdev),
        .head_offset                = offsetof(struct bpf_offload_netdev, l),
        .automatic_shrinking        = true,
};

static struct rhashtable offdevs;

static int bpf_dev_offload_check(struct net_device *netdev)
{
        if (!netdev)
                return -EINVAL;
        if (!netdev->netdev_ops->ndo_bpf)
                return -EOPNOTSUPP;
        return 0;
}

static struct bpf_offload_netdev *
bpf_offload_find_netdev(struct net_device *netdev)
{
        lockdep_assert_held(&bpf_devs_lock);

        return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
}

static int __bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
                                             struct net_device *netdev)
{
        struct bpf_offload_netdev *ondev;
        int err;

        ondev = kzalloc_obj(*ondev);
        if (!ondev)
                return -ENOMEM;

        ondev->netdev = netdev;
        ondev->offdev = offdev;
        INIT_LIST_HEAD(&ondev->progs);
        INIT_LIST_HEAD(&ondev->maps);

        err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params);
        if (err) {
                netdev_warn(netdev, "failed to register for BPF offload\n");
                goto err_free;
        }

        if (offdev)
                list_add(&ondev->offdev_netdevs, &offdev->netdevs);
        return 0;

err_free:
        kfree(ondev);
        return err;
}

static void __bpf_prog_offload_destroy(struct bpf_prog *prog)
{
        struct bpf_prog_offload *offload = prog->aux->offload;

        if (offload->dev_state)
                offload->offdev->ops->destroy(prog);

        list_del_init(&offload->offloads);
        kfree(offload);
        prog->aux->offload = NULL;
}

static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap,
                               enum bpf_netdev_command cmd)
{
        struct netdev_bpf data = {};
        struct net_device *netdev;

        ASSERT_RTNL();

        data.command = cmd;
        data.offmap = offmap;
        /* Caller must make sure netdev is valid */
        netdev = offmap->netdev;

        return netdev->netdev_ops->ndo_bpf(netdev, &data);
}

static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap)
{
        WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE));
        /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */
        bpf_map_free_id(&offmap->map);
        list_del_init(&offmap->offloads);
        offmap->netdev = NULL;
}

static void __bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
                                                struct net_device *netdev)
{
        struct bpf_offload_netdev *ondev, *altdev = NULL;
        struct bpf_offloaded_map *offmap, *mtmp;
        struct bpf_prog_offload *offload, *ptmp;

        ASSERT_RTNL();

        ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
        if (WARN_ON(!ondev))
                return;

        WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params));

        /* Try to move the objects to another netdev of the device */
        if (offdev) {
                list_del(&ondev->offdev_netdevs);
                altdev = list_first_entry_or_null(&offdev->netdevs,
                                                  struct bpf_offload_netdev,
                                                  offdev_netdevs);
        }

        if (altdev) {
                list_for_each_entry(offload, &ondev->progs, offloads)
                        offload->netdev = altdev->netdev;
                list_splice_init(&ondev->progs, &altdev->progs);

                list_for_each_entry(offmap, &ondev->maps, offloads)
                        offmap->netdev = altdev->netdev;
                list_splice_init(&ondev->maps, &altdev->maps);
        } else {
                list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads)
                        __bpf_prog_offload_destroy(offload->prog);
                list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads)
                        __bpf_map_offload_destroy(offmap);
        }

        WARN_ON(!list_empty(&ondev->progs));
        WARN_ON(!list_empty(&ondev->maps));
        kfree(ondev);
}

static int __bpf_prog_dev_bound_init(struct bpf_prog *prog, struct net_device *netdev)
{
        struct bpf_offload_netdev *ondev;
        struct bpf_prog_offload *offload;
        int err;

        offload = kzalloc_obj(*offload, GFP_USER);
        if (!offload)
                return -ENOMEM;

        offload->prog = prog;
        offload->netdev = netdev;

        ondev = bpf_offload_find_netdev(offload->netdev);
        /* When program is offloaded require presence of "true"
         * bpf_offload_netdev, avoid the one created for !ondev case below.
         */
        if (bpf_prog_is_offloaded(prog->aux) && (!ondev || !ondev->offdev)) {
                err = -EINVAL;
                goto err_free;
        }
        if (!ondev) {
                /* When only binding to the device, explicitly
                 * create an entry in the hashtable.
                 */
                err = __bpf_offload_dev_netdev_register(NULL, offload->netdev);
                if (err)
                        goto err_free;
                ondev = bpf_offload_find_netdev(offload->netdev);
        }
        offload->offdev = ondev->offdev;
        prog->aux->offload = offload;
        list_add_tail(&offload->offloads, &ondev->progs);

        return 0;
err_free:
        kfree(offload);
        return err;
}

int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr)
{
        struct net_device *netdev;
        int err;

        if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS &&
            attr->prog_type != BPF_PROG_TYPE_XDP)
                return -EINVAL;

        if (attr->prog_flags & ~(BPF_F_XDP_DEV_BOUND_ONLY | BPF_F_XDP_HAS_FRAGS))
                return -EINVAL;

        /* Frags are allowed only if program is dev-bound-only, but not
         * if it is requesting bpf offload.
         */
        if (attr->prog_flags & BPF_F_XDP_HAS_FRAGS &&
            !(attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY))
                return -EINVAL;

        if (attr->prog_type == BPF_PROG_TYPE_SCHED_CLS &&
            attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY)
                return -EINVAL;

        netdev = dev_get_by_index(current->nsproxy->net_ns, attr->prog_ifindex);
        if (!netdev)
                return -EINVAL;

        err = bpf_dev_offload_check(netdev);
        if (err)
                goto out;

        prog->aux->offload_requested = !(attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY);

        down_write(&bpf_devs_lock);
        err = __bpf_prog_dev_bound_init(prog, netdev);
        up_write(&bpf_devs_lock);

out:
        dev_put(netdev);
        return err;
}

int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog)
{
        int err;

        if (!bpf_prog_is_dev_bound(old_prog->aux))
                return 0;

        if (bpf_prog_is_offloaded(old_prog->aux))
                return -EINVAL;

        new_prog->aux->dev_bound = old_prog->aux->dev_bound;
        new_prog->aux->offload_requested = old_prog->aux->offload_requested;

        down_write(&bpf_devs_lock);
        if (!old_prog->aux->offload) {
                err = -EINVAL;
                goto out;
        }

        err = __bpf_prog_dev_bound_init(new_prog, old_prog->aux->offload->netdev);

out:
        up_write(&bpf_devs_lock);
        return err;
}

int bpf_prog_offload_verifier_prep(struct bpf_prog *prog)
{
        struct bpf_prog_offload *offload;
        int ret = -ENODEV;

        down_read(&bpf_devs_lock);
        offload = prog->aux->offload;
        if (offload) {
                ret = offload->offdev->ops->prepare(prog);
                offload->dev_state = !ret;
        }
        up_read(&bpf_devs_lock);

        return ret;
}

int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,
                                 int insn_idx, int prev_insn_idx)
{
        struct bpf_prog_offload *offload;
        int ret = -ENODEV;

        down_read(&bpf_devs_lock);
        offload = env->prog->aux->offload;
        if (offload)
                ret = offload->offdev->ops->insn_hook(env, insn_idx,
                                                      prev_insn_idx);
        up_read(&bpf_devs_lock);

        return ret;
}

int bpf_prog_offload_finalize(struct bpf_verifier_env *env)
{
        struct bpf_prog_offload *offload;
        int ret = -ENODEV;

        down_read(&bpf_devs_lock);
        offload = env->prog->aux->offload;
        if (offload) {
                if (offload->offdev->ops->finalize)
                        ret = offload->offdev->ops->finalize(env);
                else
                        ret = 0;
        }
        up_read(&bpf_devs_lock);

        return ret;
}

void
bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
                              struct bpf_insn *insn)
{
        const struct bpf_prog_offload_ops *ops;
        struct bpf_prog_offload *offload;
        int ret = -EOPNOTSUPP;

        down_read(&bpf_devs_lock);
        offload = env->prog->aux->offload;
        if (offload) {
                ops = offload->offdev->ops;
                if (!offload->opt_failed && ops->replace_insn)
                        ret = ops->replace_insn(env, off, insn);
                offload->opt_failed |= ret;
        }
        up_read(&bpf_devs_lock);
}

void
bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
{
        struct bpf_prog_offload *offload;
        int ret = -EOPNOTSUPP;

        down_read(&bpf_devs_lock);
        offload = env->prog->aux->offload;
        if (offload) {
                if (!offload->opt_failed && offload->offdev->ops->remove_insns)
                        ret = offload->offdev->ops->remove_insns(env, off, cnt);
                offload->opt_failed |= ret;
        }
        up_read(&bpf_devs_lock);
}

void bpf_prog_dev_bound_destroy(struct bpf_prog *prog)
{
        struct bpf_offload_netdev *ondev;
        struct net_device *netdev;

        rtnl_lock();
        down_write(&bpf_devs_lock);
        if (prog->aux->offload) {
                list_del_init(&prog->aux->offload->offloads);

                netdev = prog->aux->offload->netdev;
                __bpf_prog_offload_destroy(prog);

                ondev = bpf_offload_find_netdev(netdev);
                if (!ondev->offdev && list_empty(&ondev->progs))
                        __bpf_offload_dev_netdev_unregister(NULL, netdev);
        }
        up_write(&bpf_devs_lock);
        rtnl_unlock();
}

static int bpf_prog_offload_translate(struct bpf_prog *prog)
{
        struct bpf_prog_offload *offload;
        int ret = -ENODEV;

        down_read(&bpf_devs_lock);
        offload = prog->aux->offload;
        if (offload)
                ret = offload->offdev->ops->translate(prog);
        up_read(&bpf_devs_lock);

        return ret;
}

static unsigned int bpf_prog_warn_on_exec(const void *ctx,
                                          const struct bpf_insn *insn)
{
        WARN(1, "attempt to execute device eBPF program on the host!");
        return 0;
}

int bpf_prog_offload_compile(struct bpf_prog *prog)
{
        prog->bpf_func = bpf_prog_warn_on_exec;

        return bpf_prog_offload_translate(prog);
}

struct ns_get_path_bpf_prog_args {
        struct bpf_prog *prog;
        struct bpf_prog_info *info;
};

static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data)
{
        struct ns_get_path_bpf_prog_args *args = private_data;
        struct bpf_prog_aux *aux = args->prog->aux;
        struct ns_common *ns;
        struct net *net;

        rtnl_lock();
        down_read(&bpf_devs_lock);

        if (aux->offload) {
                args->info->ifindex = aux->offload->netdev->ifindex;
                net = maybe_get_net(dev_net(aux->offload->netdev));
                ns = net ? &net->ns : NULL;
        } else {
                args->info->ifindex = 0;
                ns = NULL;
        }

        up_read(&bpf_devs_lock);
        rtnl_unlock();

        return ns;
}

int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
                               struct bpf_prog *prog)
{
        struct ns_get_path_bpf_prog_args args = {
                .prog        = prog,
                .info        = info,
        };
        struct bpf_prog_aux *aux = prog->aux;
        struct inode *ns_inode;
        struct path ns_path;
        char __user *uinsns;
        int res;
        u32 ulen;

        res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args);
        if (res) {
                if (!info->ifindex)
                        return -ENODEV;
                return res;
        }

        down_read(&bpf_devs_lock);

        if (!aux->offload) {
                up_read(&bpf_devs_lock);
                return -ENODEV;
        }

        ulen = info->jited_prog_len;
        info->jited_prog_len = aux->offload->jited_len;
        if (info->jited_prog_len && ulen) {
                uinsns = u64_to_user_ptr(info->jited_prog_insns);
                ulen = min_t(u32, info->jited_prog_len, ulen);
                if (copy_to_user(uinsns, aux->offload->jited_image, ulen)) {
                        up_read(&bpf_devs_lock);
                        return -EFAULT;
                }
        }

        up_read(&bpf_devs_lock);

        ns_inode = ns_path.dentry->d_inode;
        info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev);
        info->netns_ino = ns_inode->i_ino;
        path_put(&ns_path);

        return 0;
}

const struct bpf_prog_ops bpf_offload_prog_ops = {
};

struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
{
        struct net *net = current->nsproxy->net_ns;
        struct bpf_offload_netdev *ondev;
        struct bpf_offloaded_map *offmap;
        int err;

        if (!capable(CAP_SYS_ADMIN))
                return ERR_PTR(-EPERM);
        if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
            attr->map_type != BPF_MAP_TYPE_HASH)
                return ERR_PTR(-EINVAL);

        offmap = bpf_map_area_alloc(sizeof(*offmap), NUMA_NO_NODE);
        if (!offmap)
                return ERR_PTR(-ENOMEM);

        bpf_map_init_from_attr(&offmap->map, attr);
        rtnl_lock();
        offmap->netdev = __dev_get_by_index(net, attr->map_ifindex);
        err = bpf_dev_offload_check(offmap->netdev);
        if (err)
                goto err_unlock_rtnl;

        netdev_lock_ops(offmap->netdev);
        down_write(&bpf_devs_lock);

        ondev = bpf_offload_find_netdev(offmap->netdev);
        if (!ondev) {
                err = -EINVAL;
                goto err_unlock;
        }

        err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC);
        if (err)
                goto err_unlock;

        list_add_tail(&offmap->offloads, &ondev->maps);
        up_write(&bpf_devs_lock);
        netdev_unlock_ops(offmap->netdev);
        rtnl_unlock();

        return &offmap->map;

err_unlock:
        up_write(&bpf_devs_lock);
        netdev_unlock_ops(offmap->netdev);
err_unlock_rtnl:
        rtnl_unlock();
        bpf_map_area_free(offmap);
        return ERR_PTR(err);
}

void bpf_map_offload_map_free(struct bpf_map *map)
{
        struct bpf_offloaded_map *offmap = map_to_offmap(map);

        rtnl_lock();
        down_write(&bpf_devs_lock);
        if (offmap->netdev)
                __bpf_map_offload_destroy(offmap);
        up_write(&bpf_devs_lock);
        rtnl_unlock();

        bpf_map_area_free(offmap);
}

u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
{
        /* The memory dynamically allocated in netdev dev_ops is not counted */
        return sizeof(struct bpf_offloaded_map);
}

int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value)
{
        struct bpf_offloaded_map *offmap = map_to_offmap(map);
        int ret = -ENODEV;

        down_read(&bpf_devs_lock);
        if (offmap->netdev)
                ret = offmap->dev_ops->map_lookup_elem(offmap, key, value);
        up_read(&bpf_devs_lock);

        return ret;
}

int bpf_map_offload_update_elem(struct bpf_map *map,
                                void *key, void *value, u64 flags)
{
        struct bpf_offloaded_map *offmap = map_to_offmap(map);
        int ret = -ENODEV;

        if (unlikely(flags > BPF_EXIST))
                return -EINVAL;

        down_read(&bpf_devs_lock);
        if (offmap->netdev)
                ret = offmap->dev_ops->map_update_elem(offmap, key, value,
                                                       flags);
        up_read(&bpf_devs_lock);

        return ret;
}

int bpf_map_offload_delete_elem(struct bpf_map *map, void *key)
{
        struct bpf_offloaded_map *offmap = map_to_offmap(map);
        int ret = -ENODEV;

        down_read(&bpf_devs_lock);
        if (offmap->netdev)
                ret = offmap->dev_ops->map_delete_elem(offmap, key);
        up_read(&bpf_devs_lock);

        return ret;
}

int bpf_map_offload_get_next_key(struct bpf_map *map, void *key, void *next_key)
{
        struct bpf_offloaded_map *offmap = map_to_offmap(map);
        int ret = -ENODEV;

        down_read(&bpf_devs_lock);
        if (offmap->netdev)
                ret = offmap->dev_ops->map_get_next_key(offmap, key, next_key);
        up_read(&bpf_devs_lock);

        return ret;
}

struct ns_get_path_bpf_map_args {
        struct bpf_offloaded_map *offmap;
        struct bpf_map_info *info;
};

static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data)
{
        struct ns_get_path_bpf_map_args *args = private_data;
        struct ns_common *ns;
        struct net *net;

        rtnl_lock();
        down_read(&bpf_devs_lock);

        if (args->offmap->netdev) {
                args->info->ifindex = args->offmap->netdev->ifindex;
                net = maybe_get_net(dev_net(args->offmap->netdev));
                ns = net ? &net->ns : NULL;
        } else {
                args->info->ifindex = 0;
                ns = NULL;
        }

        up_read(&bpf_devs_lock);
        rtnl_unlock();

        return ns;
}

int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map)
{
        struct ns_get_path_bpf_map_args args = {
                .offmap        = map_to_offmap(map),
                .info        = info,
        };
        struct inode *ns_inode;
        struct path ns_path;
        int res;

        res = ns_get_path_cb(&ns_path, bpf_map_offload_info_fill_ns, &args);
        if (res) {
                if (!info->ifindex)
                        return -ENODEV;
                return res;
        }

        ns_inode = ns_path.dentry->d_inode;
        info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev);
        info->netns_ino = ns_inode->i_ino;
        path_put(&ns_path);

        return 0;
}

static bool __bpf_offload_dev_match(struct bpf_prog *prog,
                                    struct net_device *netdev)
{
        struct bpf_offload_netdev *ondev1, *ondev2;
        struct bpf_prog_offload *offload;

        if (!bpf_prog_is_dev_bound(prog->aux))
                return false;

        offload = prog->aux->offload;
        if (!offload)
                return false;
        if (offload->netdev == netdev)
                return true;

        ondev1 = bpf_offload_find_netdev(offload->netdev);
        ondev2 = bpf_offload_find_netdev(netdev);

        return ondev1 && ondev2 && ondev1->offdev == ondev2->offdev;
}

bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev)
{
        bool ret;

        down_read(&bpf_devs_lock);
        ret = __bpf_offload_dev_match(prog, netdev);
        up_read(&bpf_devs_lock);

        return ret;
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_match);

bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)
{
        bool ret;

        if (bpf_prog_is_offloaded(lhs->aux) != bpf_prog_is_offloaded(rhs->aux))
                return false;

        down_read(&bpf_devs_lock);
        ret = lhs->aux->offload && rhs->aux->offload &&
              lhs->aux->offload->netdev &&
              lhs->aux->offload->netdev == rhs->aux->offload->netdev;
        up_read(&bpf_devs_lock);

        return ret;
}

bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map)
{
        struct bpf_offloaded_map *offmap;
        bool ret;

        if (!bpf_map_is_offloaded(map))
                return bpf_map_offload_neutral(map);
        offmap = map_to_offmap(map);

        down_read(&bpf_devs_lock);
        ret = __bpf_offload_dev_match(prog, offmap->netdev);
        up_read(&bpf_devs_lock);

        return ret;
}

int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
                                    struct net_device *netdev)
{
        int err;

        down_write(&bpf_devs_lock);
        err = __bpf_offload_dev_netdev_register(offdev, netdev);
        up_write(&bpf_devs_lock);
        return err;
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register);

void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
                                       struct net_device *netdev)
{
        down_write(&bpf_devs_lock);
        __bpf_offload_dev_netdev_unregister(offdev, netdev);
        up_write(&bpf_devs_lock);
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister);

struct bpf_offload_dev *
bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv)
{
        struct bpf_offload_dev *offdev;

        offdev = kzalloc_obj(*offdev);
        if (!offdev)
                return ERR_PTR(-ENOMEM);

        offdev->ops = ops;
        offdev->priv = priv;
        INIT_LIST_HEAD(&offdev->netdevs);

        return offdev;
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_create);

void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev)
{
        WARN_ON(!list_empty(&offdev->netdevs));
        kfree(offdev);
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy);

void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev)
{
        return offdev->priv;
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_priv);

void bpf_dev_bound_netdev_unregister(struct net_device *dev)
{
        struct bpf_offload_netdev *ondev;

        ASSERT_RTNL();

        down_write(&bpf_devs_lock);
        ondev = bpf_offload_find_netdev(dev);
        if (ondev && !ondev->offdev)
                __bpf_offload_dev_netdev_unregister(NULL, ondev->netdev);
        up_write(&bpf_devs_lock);
}

int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
                              struct bpf_prog_aux *prog_aux)
{
        if (!bpf_prog_is_dev_bound(prog_aux)) {
                bpf_log(log, "metadata kfuncs require device-bound program\n");
                return -EINVAL;
        }

        if (bpf_prog_is_offloaded(prog_aux)) {
                bpf_log(log, "metadata kfuncs can't be offloaded\n");
                return -EINVAL;
        }

        return 0;
}

void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id)
{
        const struct xdp_metadata_ops *ops;
        void *p = NULL;

        /* We don't hold bpf_devs_lock while resolving several
         * kfuncs and can race with the unregister_netdevice().
         * We rely on bpf_dev_bound_match() check at attach
         * to render this program unusable.
         */
        down_read(&bpf_devs_lock);
        if (!prog->aux->offload)
                goto out;

        ops = prog->aux->offload->netdev->xdp_metadata_ops;
        if (!ops)
                goto out;

#define XDP_METADATA_KFUNC(name, _, __, xmo) \
        if (func_id == bpf_xdp_metadata_kfunc_id(name)) p = ops->xmo;
        XDP_METADATA_KFUNC_xxx
#undef XDP_METADATA_KFUNC

out:
        up_read(&bpf_devs_lock);

        return p;
}

static int __init bpf_offload_init(void)
{
        return rhashtable_init(&offdevs, &offdevs_params);
}

core_initcall(bpf_offload_init);


















 4967 


 5796 

















    1 



    1 






































































































   73 





























   96 


















































  253 
















  253 




















































































 1370 




 1370 














 3069 









































































































  245 














 1069 














 1184 














 19885 

























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
// SPDX-License-Identifier: GPL-2.0
/*
 * security/tomoyo/tomoyo.c
 *
 * Copyright (C) 2005-2011  NTT DATA CORPORATION
 */

#include <linux/lsm_hooks.h>
#include <uapi/linux/lsm.h>
#include "common.h"

/**
 * tomoyo_domain - Get "struct tomoyo_domain_info" for current thread.
 *
 * Returns pointer to "struct tomoyo_domain_info" for current thread.
 */
struct tomoyo_domain_info *tomoyo_domain(void)
{
        struct tomoyo_task *s = tomoyo_task(current);

        if (s->old_domain_info && !current->in_execve) {
                atomic_dec(&s->old_domain_info->users);
                s->old_domain_info = NULL;
        }
        return s->domain_info;
}

/**
 * tomoyo_cred_prepare - Target for security_prepare_creds().
 *
 * @new: Pointer to "struct cred".
 * @old: Pointer to "struct cred".
 * @gfp: Memory allocation flags.
 *
 * Returns 0.
 */
static int tomoyo_cred_prepare(struct cred *new, const struct cred *old,
                               gfp_t gfp)
{
        /* Restore old_domain_info saved by previous execve() request. */
        struct tomoyo_task *s = tomoyo_task(current);

        if (s->old_domain_info && !current->in_execve) {
                atomic_dec(&s->domain_info->users);
                s->domain_info = s->old_domain_info;
                s->old_domain_info = NULL;
        }
        return 0;
}

/**
 * tomoyo_bprm_committed_creds - Target for security_bprm_committed_creds().
 *
 * @bprm: Pointer to "struct linux_binprm".
 */
static void tomoyo_bprm_committed_creds(const struct linux_binprm *bprm)
{
        /* Clear old_domain_info saved by execve() request. */
        struct tomoyo_task *s = tomoyo_task(current);

        atomic_dec(&s->old_domain_info->users);
        s->old_domain_info = NULL;
}

#ifndef CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER
/**
 * tomoyo_bprm_creds_for_exec - Target for security_bprm_creds_for_exec().
 *
 * @bprm: Pointer to "struct linux_binprm".
 *
 * Returns 0.
 */
static int tomoyo_bprm_creds_for_exec(struct linux_binprm *bprm)
{
        /*
         * Load policy if /sbin/tomoyo-init exists and /sbin/init is requested
         * for the first time.
         */
        if (!tomoyo_policy_loaded)
                tomoyo_load_policy(bprm->filename);
        return 0;
}
#endif

/**
 * tomoyo_bprm_check_security - Target for security_bprm_check().
 *
 * @bprm: Pointer to "struct linux_binprm".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_bprm_check_security(struct linux_binprm *bprm)
{
        struct tomoyo_task *s = tomoyo_task(current);

        /*
         * Execute permission is checked against pathname passed to execve()
         * using current domain.
         */
        if (!s->old_domain_info) {
                const int idx = tomoyo_read_lock();
                const int err = tomoyo_find_next_domain(bprm);

                tomoyo_read_unlock(idx);
                return err;
        }
        /*
         * Read permission is checked against interpreters using next domain.
         */
        return tomoyo_check_open_permission(s->domain_info,
                                            &bprm->file->f_path, O_RDONLY);
}

/**
 * tomoyo_inode_getattr - Target for security_inode_getattr().
 *
 * @path: Pointer to "struct path".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_inode_getattr(const struct path *path)
{
        return tomoyo_path_perm(TOMOYO_TYPE_GETATTR, path, NULL);
}

/**
 * tomoyo_path_truncate - Target for security_path_truncate().
 *
 * @path: Pointer to "struct path".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_truncate(const struct path *path)
{
        return tomoyo_path_perm(TOMOYO_TYPE_TRUNCATE, path, NULL);
}

/**
 * tomoyo_file_truncate - Target for security_file_truncate().
 *
 * @file: Pointer to "struct file".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_file_truncate(struct file *file)
{
        return tomoyo_path_truncate(&file->f_path);
}

/**
 * tomoyo_path_unlink - Target for security_path_unlink().
 *
 * @parent: Pointer to "struct path".
 * @dentry: Pointer to "struct dentry".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_unlink(const struct path *parent, struct dentry *dentry)
{
        struct path path = { .mnt = parent->mnt, .dentry = dentry };

        return tomoyo_path_perm(TOMOYO_TYPE_UNLINK, &path, NULL);
}

/**
 * tomoyo_path_mkdir - Target for security_path_mkdir().
 *
 * @parent: Pointer to "struct path".
 * @dentry: Pointer to "struct dentry".
 * @mode:   DAC permission mode.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_mkdir(const struct path *parent, struct dentry *dentry,
                             umode_t mode)
{
        struct path path = { .mnt = parent->mnt, .dentry = dentry };

        return tomoyo_path_number_perm(TOMOYO_TYPE_MKDIR, &path,
                                       mode & S_IALLUGO);
}

/**
 * tomoyo_path_rmdir - Target for security_path_rmdir().
 *
 * @parent: Pointer to "struct path".
 * @dentry: Pointer to "struct dentry".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_rmdir(const struct path *parent, struct dentry *dentry)
{
        struct path path = { .mnt = parent->mnt, .dentry = dentry };

        return tomoyo_path_perm(TOMOYO_TYPE_RMDIR, &path, NULL);
}

/**
 * tomoyo_path_symlink - Target for security_path_symlink().
 *
 * @parent:   Pointer to "struct path".
 * @dentry:   Pointer to "struct dentry".
 * @old_name: Symlink's content.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_symlink(const struct path *parent, struct dentry *dentry,
                               const char *old_name)
{
        struct path path = { .mnt = parent->mnt, .dentry = dentry };

        return tomoyo_path_perm(TOMOYO_TYPE_SYMLINK, &path, old_name);
}

/**
 * tomoyo_path_mknod - Target for security_path_mknod().
 *
 * @parent: Pointer to "struct path".
 * @dentry: Pointer to "struct dentry".
 * @mode:   DAC permission mode.
 * @dev:    Device attributes.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_mknod(const struct path *parent, struct dentry *dentry,
                             umode_t mode, unsigned int dev)
{
        struct path path = { .mnt = parent->mnt, .dentry = dentry };
        int type = TOMOYO_TYPE_CREATE;
        const unsigned int perm = mode & S_IALLUGO;

        switch (mode & S_IFMT) {
        case S_IFCHR:
                type = TOMOYO_TYPE_MKCHAR;
                break;
        case S_IFBLK:
                type = TOMOYO_TYPE_MKBLOCK;
                break;
        default:
                goto no_dev;
        }
        return tomoyo_mkdev_perm(type, &path, perm, dev);
 no_dev:
        switch (mode & S_IFMT) {
        case S_IFIFO:
                type = TOMOYO_TYPE_MKFIFO;
                break;
        case S_IFSOCK:
                type = TOMOYO_TYPE_MKSOCK;
                break;
        }
        return tomoyo_path_number_perm(type, &path, perm);
}

/**
 * tomoyo_path_link - Target for security_path_link().
 *
 * @old_dentry: Pointer to "struct dentry".
 * @new_dir:    Pointer to "struct path".
 * @new_dentry: Pointer to "struct dentry".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_link(struct dentry *old_dentry, const struct path *new_dir,
                            struct dentry *new_dentry)
{
        struct path path1 = { .mnt = new_dir->mnt, .dentry = old_dentry };
        struct path path2 = { .mnt = new_dir->mnt, .dentry = new_dentry };

        return tomoyo_path2_perm(TOMOYO_TYPE_LINK, &path1, &path2);
}

/**
 * tomoyo_path_rename - Target for security_path_rename().
 *
 * @old_parent: Pointer to "struct path".
 * @old_dentry: Pointer to "struct dentry".
 * @new_parent: Pointer to "struct path".
 * @new_dentry: Pointer to "struct dentry".
 * @flags: Rename options.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_rename(const struct path *old_parent,
                              struct dentry *old_dentry,
                              const struct path *new_parent,
                              struct dentry *new_dentry,
                              const unsigned int flags)
{
        struct path path1 = { .mnt = old_parent->mnt, .dentry = old_dentry };
        struct path path2 = { .mnt = new_parent->mnt, .dentry = new_dentry };

        if (flags & RENAME_EXCHANGE) {
                const int err = tomoyo_path2_perm(TOMOYO_TYPE_RENAME, &path2,
                                &path1);

                if (err)
                        return err;
        }
        return tomoyo_path2_perm(TOMOYO_TYPE_RENAME, &path1, &path2);
}

/**
 * tomoyo_file_fcntl - Target for security_file_fcntl().
 *
 * @file: Pointer to "struct file".
 * @cmd:  Command for fcntl().
 * @arg:  Argument for @cmd.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_file_fcntl(struct file *file, unsigned int cmd,
                             unsigned long arg)
{
        if (!(cmd == F_SETFL && ((arg ^ file->f_flags) & O_APPEND)))
                return 0;
        return tomoyo_check_open_permission(tomoyo_domain(), &file->f_path,
                                            O_WRONLY | (arg & O_APPEND));
}

/**
 * tomoyo_file_open - Target for security_file_open().
 *
 * @f: Pointer to "struct file".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_file_open(struct file *f)
{
        /* Don't check read permission here if called from execve(). */
        /* Illogically, FMODE_EXEC is in f_flags, not f_mode. */
        if (f->f_flags & __FMODE_EXEC)
                return 0;
        return tomoyo_check_open_permission(tomoyo_domain(), &f->f_path,
                                            f->f_flags);
}

/**
 * tomoyo_file_ioctl - Target for security_file_ioctl().
 *
 * @file: Pointer to "struct file".
 * @cmd:  Command for ioctl().
 * @arg:  Argument for @cmd.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_file_ioctl(struct file *file, unsigned int cmd,
                             unsigned long arg)
{
        return tomoyo_path_number_perm(TOMOYO_TYPE_IOCTL, &file->f_path, cmd);
}

/**
 * tomoyo_path_chmod - Target for security_path_chmod().
 *
 * @path: Pointer to "struct path".
 * @mode: DAC permission mode.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_chmod(const struct path *path, umode_t mode)
{
        return tomoyo_path_number_perm(TOMOYO_TYPE_CHMOD, path,
                                       mode & S_IALLUGO);
}

/**
 * tomoyo_path_chown - Target for security_path_chown().
 *
 * @path: Pointer to "struct path".
 * @uid:  Owner ID.
 * @gid:  Group ID.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_chown(const struct path *path, kuid_t uid, kgid_t gid)
{
        int error = 0;

        if (uid_valid(uid))
                error = tomoyo_path_number_perm(TOMOYO_TYPE_CHOWN, path,
                                                from_kuid(&init_user_ns, uid));
        if (!error && gid_valid(gid))
                error = tomoyo_path_number_perm(TOMOYO_TYPE_CHGRP, path,
                                                from_kgid(&init_user_ns, gid));
        return error;
}

/**
 * tomoyo_path_chroot - Target for security_path_chroot().
 *
 * @path: Pointer to "struct path".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_path_chroot(const struct path *path)
{
        return tomoyo_path_perm(TOMOYO_TYPE_CHROOT, path, NULL);
}

/**
 * tomoyo_sb_mount - Target for security_sb_mount().
 *
 * @dev_name: Name of device file. Maybe NULL.
 * @path:     Pointer to "struct path".
 * @type:     Name of filesystem type. Maybe NULL.
 * @flags:    Mount options.
 * @data:     Optional data. Maybe NULL.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_sb_mount(const char *dev_name, const struct path *path,
                           const char *type, unsigned long flags, void *data)
{
        return tomoyo_mount_permission(dev_name, path, type, flags, data);
}

/**
 * tomoyo_sb_umount - Target for security_sb_umount().
 *
 * @mnt:   Pointer to "struct vfsmount".
 * @flags: Unmount options.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_sb_umount(struct vfsmount *mnt, int flags)
{
        struct path path = { .mnt = mnt, .dentry = mnt->mnt_root };

        return tomoyo_path_perm(TOMOYO_TYPE_UMOUNT, &path, NULL);
}

/**
 * tomoyo_sb_pivotroot - Target for security_sb_pivotroot().
 *
 * @old_path: Pointer to "struct path".
 * @new_path: Pointer to "struct path".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_sb_pivotroot(const struct path *old_path, const struct path *new_path)
{
        return tomoyo_path2_perm(TOMOYO_TYPE_PIVOT_ROOT, new_path, old_path);
}

/**
 * tomoyo_socket_listen - Check permission for listen().
 *
 * @sock:    Pointer to "struct socket".
 * @backlog: Backlog parameter.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_socket_listen(struct socket *sock, int backlog)
{
        return tomoyo_socket_listen_permission(sock);
}

/**
 * tomoyo_socket_connect - Check permission for connect().
 *
 * @sock:     Pointer to "struct socket".
 * @addr:     Pointer to "struct sockaddr".
 * @addr_len: Size of @addr.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_socket_connect(struct socket *sock, struct sockaddr *addr,
                                 int addr_len)
{
        return tomoyo_socket_connect_permission(sock, addr, addr_len);
}

/**
 * tomoyo_socket_bind - Check permission for bind().
 *
 * @sock:     Pointer to "struct socket".
 * @addr:     Pointer to "struct sockaddr".
 * @addr_len: Size of @addr.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_socket_bind(struct socket *sock, struct sockaddr *addr,
                              int addr_len)
{
        return tomoyo_socket_bind_permission(sock, addr, addr_len);
}

/**
 * tomoyo_socket_sendmsg - Check permission for sendmsg().
 *
 * @sock: Pointer to "struct socket".
 * @msg:  Pointer to "struct msghdr".
 * @size: Size of message.
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_socket_sendmsg(struct socket *sock, struct msghdr *msg,
                                 int size)
{
        return tomoyo_socket_sendmsg_permission(sock, msg, size);
}

struct lsm_blob_sizes tomoyo_blob_sizes __ro_after_init = {
        .lbs_task = sizeof(struct tomoyo_task),
};

/**
 * tomoyo_task_alloc - Target for security_task_alloc().
 *
 * @task:        Pointer to "struct task_struct".
 * @clone_flags: clone() flags.
 *
 * Returns 0.
 */
static int tomoyo_task_alloc(struct task_struct *task,
                             u64 clone_flags)
{
        struct tomoyo_task *old = tomoyo_task(current);
        struct tomoyo_task *new = tomoyo_task(task);

        new->domain_info = old->domain_info;
        atomic_inc(&new->domain_info->users);
        new->old_domain_info = NULL;
        return 0;
}

/**
 * tomoyo_task_free - Target for security_task_free().
 *
 * @task: Pointer to "struct task_struct".
 */
static void tomoyo_task_free(struct task_struct *task)
{
        struct tomoyo_task *s = tomoyo_task(task);

        if (s->domain_info) {
                atomic_dec(&s->domain_info->users);
                s->domain_info = NULL;
        }
        if (s->old_domain_info) {
                atomic_dec(&s->old_domain_info->users);
                s->old_domain_info = NULL;
        }
}

static const struct lsm_id tomoyo_lsmid = {
        .name = "tomoyo",
        .id = LSM_ID_TOMOYO,
};

/* tomoyo_hooks is used for registering TOMOYO. */
static struct security_hook_list tomoyo_hooks[] __ro_after_init = {
        LSM_HOOK_INIT(cred_prepare, tomoyo_cred_prepare),
        LSM_HOOK_INIT(bprm_committed_creds, tomoyo_bprm_committed_creds),
        LSM_HOOK_INIT(task_alloc, tomoyo_task_alloc),
        LSM_HOOK_INIT(task_free, tomoyo_task_free),
#ifndef CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER
        LSM_HOOK_INIT(bprm_creds_for_exec, tomoyo_bprm_creds_for_exec),
#endif
        LSM_HOOK_INIT(bprm_check_security, tomoyo_bprm_check_security),
        LSM_HOOK_INIT(file_fcntl, tomoyo_file_fcntl),
        LSM_HOOK_INIT(file_open, tomoyo_file_open),
        LSM_HOOK_INIT(file_truncate, tomoyo_file_truncate),
        LSM_HOOK_INIT(path_truncate, tomoyo_path_truncate),
        LSM_HOOK_INIT(path_unlink, tomoyo_path_unlink),
        LSM_HOOK_INIT(path_mkdir, tomoyo_path_mkdir),
        LSM_HOOK_INIT(path_rmdir, tomoyo_path_rmdir),
        LSM_HOOK_INIT(path_symlink, tomoyo_path_symlink),
        LSM_HOOK_INIT(path_mknod, tomoyo_path_mknod),
        LSM_HOOK_INIT(path_link, tomoyo_path_link),
        LSM_HOOK_INIT(path_rename, tomoyo_path_rename),
        LSM_HOOK_INIT(inode_getattr, tomoyo_inode_getattr),
        LSM_HOOK_INIT(file_ioctl, tomoyo_file_ioctl),
        LSM_HOOK_INIT(file_ioctl_compat, tomoyo_file_ioctl),
        LSM_HOOK_INIT(path_chmod, tomoyo_path_chmod),
        LSM_HOOK_INIT(path_chown, tomoyo_path_chown),
        LSM_HOOK_INIT(path_chroot, tomoyo_path_chroot),
        LSM_HOOK_INIT(sb_mount, tomoyo_sb_mount),
        LSM_HOOK_INIT(sb_umount, tomoyo_sb_umount),
        LSM_HOOK_INIT(sb_pivotroot, tomoyo_sb_pivotroot),
        LSM_HOOK_INIT(socket_bind, tomoyo_socket_bind),
        LSM_HOOK_INIT(socket_connect, tomoyo_socket_connect),
        LSM_HOOK_INIT(socket_listen, tomoyo_socket_listen),
        LSM_HOOK_INIT(socket_sendmsg, tomoyo_socket_sendmsg),
};

/* Lock for GC. */
DEFINE_SRCU(tomoyo_ss);

int tomoyo_enabled __ro_after_init = 1;

/**
 * tomoyo_init - Register TOMOYO Linux as a LSM module.
 *
 * Returns 0.
 */
static int __init tomoyo_init(void)
{
        struct tomoyo_task *s = tomoyo_task(current);

        /* register ourselves with the security framework */
        security_add_hooks(tomoyo_hooks, ARRAY_SIZE(tomoyo_hooks),
                           &tomoyo_lsmid);
        pr_info("TOMOYO Linux initialized\n");
        s->domain_info = &tomoyo_kernel_domain;
        atomic_inc(&tomoyo_kernel_domain.users);
        s->old_domain_info = NULL;
        tomoyo_mm_init();

        return 0;
}

DEFINE_LSM(tomoyo) = {
        .id = &tomoyo_lsmid,
        .enabled = &tomoyo_enabled,
        .flags = LSM_FLAG_LEGACY_MAJOR,
        .blobs = &tomoyo_blob_sizes,
        .init = tomoyo_init,
        .initcall_fs = tomoyo_interface_init,
};

























































































































































































































   41 




















































































 26510 









































































































































































    2 




 26510 




 26510 















































  243 
   15 





































  243 
  243 








































   47 
















   33 























































































































































































































































































































































 1105 




 1105 


 1105 


 1105 

    2 


 1105 




 26096 





 26096 


 26096 

 26096 

   23 

 26096 



























 5570 

 5570 




 5570 


 5570 
 5570 
 5570 

    6 







  119 

  119 




  119 


  119 

  119 













   13 
   13 


















































  274 





  274 


  274 

  274 


    5 



  274 













  286 


  286 















 6282 




































































































































































































































































































































































































































































































































































































































































































































































    1 





    1 

    1 




    1 





























































































































































































































































































































































































































































































































































































































































































































































































   26 




   26 



   26 












 4215 






 4215 


    4 



 4187 
 1729 































































   41 





   41 

   41 

    1 






























  196 






  196 


  196 


  196 







  196 




  196 





















































































































































































































































































    2 






    2 






    2 
    2 






















    2 

















    2 
    2 










































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
// SPDX-License-Identifier: GPL-2.0
/*
 *  Kernel timekeeping code and accessor functions. Based on code from
 *  timer.c, moved in commit 8524070b7982.
 */
#include <linux/audit.h>
#include <linux/clocksource.h>
#include <linux/compiler.h>
#include <linux/jiffies.h>
#include <linux/kobject.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/pvclock_gtod.h>
#include <linux/random.h>
#include <linux/sched/clock.h>
#include <linux/sched/loadavg.h>
#include <linux/static_key.h>
#include <linux/stop_machine.h>
#include <linux/syscore_ops.h>
#include <linux/tick.h>
#include <linux/time.h>
#include <linux/timex.h>
#include <linux/timekeeper_internal.h>

#include <vdso/auxclock.h>

#include "tick-internal.h"
#include "timekeeping_internal.h"
#include "ntp_internal.h"

#define TK_CLEAR_NTP                (1 << 0)
#define TK_CLOCK_WAS_SET        (1 << 1)

#define TK_UPDATE_ALL                (TK_CLEAR_NTP | TK_CLOCK_WAS_SET)

enum timekeeping_adv_mode {
        /* Update timekeeper when a tick has passed */
        TK_ADV_TICK,

        /* Update timekeeper on a direct frequency change */
        TK_ADV_FREQ
};

/*
 * The most important data for readout fits into a single 64 byte
 * cache line.
 */
struct tk_data {
        seqcount_raw_spinlock_t        seq;
        struct timekeeper        timekeeper;
        struct timekeeper        shadow_timekeeper;
        raw_spinlock_t                lock;
} ____cacheline_aligned;

static struct tk_data timekeeper_data[TIMEKEEPERS_MAX];

/* The core timekeeper */
#define tk_core                (timekeeper_data[TIMEKEEPER_CORE])

#ifdef CONFIG_POSIX_AUX_CLOCKS
static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts)
{
        return ktime_get_aux_ts64(CLOCK_AUX + tkid - TIMEKEEPER_AUX_FIRST, ts);
}

static inline bool tk_is_aux(const struct timekeeper *tk)
{
        return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST;
}
static inline struct tk_data *aux_get_tk_data(clockid_t id);
#else
static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts)
{
        return false;
}

static inline bool tk_is_aux(const struct timekeeper *tk)
{
        return false;
}
static inline struct tk_data *aux_get_tk_data(clockid_t id)
{
        return NULL;
}
#endif

static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs)
{
        tk->offs_aux = offs;
        tk->monotonic_to_aux = ktime_to_timespec64(offs);
}

/* flag for if timekeeping is suspended */
int __read_mostly timekeeping_suspended;

/**
 * struct tk_fast - NMI safe timekeeper
 * @seq:        Sequence counter for protecting updates. The lowest bit
 *                is the index for the tk_read_base array
 * @base:        tk_read_base array. Access is indexed by the lowest bit of
 *                @seq.
 *
 * See @update_fast_timekeeper() below.
 */
struct tk_fast {
        seqcount_latch_t        seq;
        struct tk_read_base        base[2];
};

/* Suspend-time cycles value for halted fast timekeeper. */
static u64 cycles_at_suspend;

static u64 dummy_clock_read(struct clocksource *cs)
{
        if (timekeeping_suspended)
                return cycles_at_suspend;
        return local_clock();
}

static struct clocksource dummy_clock = {
        .read = dummy_clock_read,
};

/*
 * Boot time initialization which allows local_clock() to be utilized
 * during early boot when clocksources are not available. local_clock()
 * returns nanoseconds already so no conversion is required, hence mult=1
 * and shift=0. When the first proper clocksource is installed then
 * the fast time keepers are updated with the correct values.
 */
#define FAST_TK_INIT                                                \
        {                                                        \
                .clock                = &dummy_clock,                        \
                .mask                = CLOCKSOURCE_MASK(64),                \
                .mult                = 1,                                \
                .shift                = 0,                                \
        }

static struct tk_fast tk_fast_mono ____cacheline_aligned = {
        .seq     = SEQCNT_LATCH_ZERO(tk_fast_mono.seq),
        .base[0] = FAST_TK_INIT,
        .base[1] = FAST_TK_INIT,
};

static struct tk_fast tk_fast_raw  ____cacheline_aligned = {
        .seq     = SEQCNT_LATCH_ZERO(tk_fast_raw.seq),
        .base[0] = FAST_TK_INIT,
        .base[1] = FAST_TK_INIT,
};

#ifdef CONFIG_POSIX_AUX_CLOCKS
static __init void tk_aux_setup(void);
static void tk_aux_update_clocksource(void);
static void tk_aux_advance(void);
#else
static inline void tk_aux_setup(void) { }
static inline void tk_aux_update_clocksource(void) { }
static inline void tk_aux_advance(void) { }
#endif

unsigned long timekeeper_lock_irqsave(void)
{
        unsigned long flags;

        raw_spin_lock_irqsave(&tk_core.lock, flags);
        return flags;
}

void timekeeper_unlock_irqrestore(unsigned long flags)
{
        raw_spin_unlock_irqrestore(&tk_core.lock, flags);
}

/*
 * Multigrain timestamps require tracking the latest fine-grained timestamp
 * that has been issued, and never returning a coarse-grained timestamp that is
 * earlier than that value.
 *
 * mg_floor represents the latest fine-grained time that has been handed out as
 * a file timestamp on the system. This is tracked as a monotonic ktime_t, and
 * converted to a realtime clock value on an as-needed basis.
 *
 * Maintaining mg_floor ensures the multigrain interfaces never issue a
 * timestamp earlier than one that has been previously issued.
 *
 * The exception to this rule is when there is a backward realtime clock jump. If
 * such an event occurs, a timestamp can appear to be earlier than a previous one.
 */
static __cacheline_aligned_in_smp atomic64_t mg_floor;

static inline void tk_normalize_xtime(struct timekeeper *tk)
{
        while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
                tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
                tk->xtime_sec++;
        }
        while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {
                tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
                tk->raw_sec++;
        }
}

static inline struct timespec64 tk_xtime(const struct timekeeper *tk)
{
        struct timespec64 ts;

        ts.tv_sec = tk->xtime_sec;
        ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
        return ts;
}

static inline struct timespec64 tk_xtime_coarse(const struct timekeeper *tk)
{
        struct timespec64 ts;

        ts.tv_sec = tk->xtime_sec;
        ts.tv_nsec = tk->coarse_nsec;
        return ts;
}

/*
 * Update the nanoseconds part for the coarse time keepers. They can't rely
 * on xtime_nsec because xtime_nsec could be adjusted by a small negative
 * amount when the multiplication factor of the clock is adjusted, which
 * could cause the coarse clocks to go slightly backwards. See
 * timekeeping_apply_adjustment(). Thus we keep a separate copy for the coarse
 * clockids which only is updated when the clock has been set or  we have
 * accumulated time.
 */
static inline void tk_update_coarse_nsecs(struct timekeeper *tk)
{
        tk->coarse_nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
}

static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
{
        tk->xtime_sec = ts->tv_sec;
        tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
        tk_update_coarse_nsecs(tk);
}

static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
{
        tk->xtime_sec += ts->tv_sec;
        tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
        tk_normalize_xtime(tk);
        tk_update_coarse_nsecs(tk);
}

static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
{
        struct timespec64 tmp;

        /*
         * Verify consistency of: offset_real = -wall_to_monotonic
         * before modifying anything
         */
        set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
                                        -tk->wall_to_monotonic.tv_nsec);
        WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
        tk->wall_to_monotonic = wtm;
        set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
        /* Paired with READ_ONCE() in ktime_mono_to_any() */
        WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp));
        WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)));
}

static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
{
        /* Paired with READ_ONCE() in ktime_mono_to_any() */
        WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta));
        /*
         * Timespec representation for VDSO update to avoid 64bit division
         * on every update.
         */
        tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);
}

#ifdef CONFIG_ARCH_WANTS_CLOCKSOURCE_READ_INLINE
#include <asm/clock_inlined.h>

static DEFINE_STATIC_KEY_FALSE(clocksource_read_inlined);

/*
 * tk_clock_read - atomic clocksource read() helper
 *
 * This helper is necessary to use in the read paths because, while the
 * seqcount ensures we don't return a bad value while structures are updated,
 * it doesn't protect from potential crashes. There is the possibility that
 * the tkr's clocksource may change between the read reference, and the
 * clock reference passed to the read function.  This can cause crashes if
 * the wrong clocksource is passed to the wrong read function.
 * This isn't necessary to use when holding the tk_core.lock or doing
 * a read of the fast-timekeeper tkrs (which is protected by its own locking
 * and update logic).
 */
static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr)
{
        struct clocksource *clock = READ_ONCE(tkr->clock);

        if (static_branch_likely(&clocksource_read_inlined))
                return arch_inlined_clocksource_read(clock);

        return clock->read(clock);
}

static inline void clocksource_disable_inline_read(void)
{
        static_branch_disable(&clocksource_read_inlined);
}

static inline void clocksource_enable_inline_read(void)
{
        static_branch_enable(&clocksource_read_inlined);
}
#else
static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr)
{
        struct clocksource *clock = READ_ONCE(tkr->clock);

        return clock->read(clock);
}

static inline void clocksource_disable_inline_read(void) { }
static inline void clocksource_enable_inline_read(void) { }
#endif

/**
 * tk_setup_internals - Set up internals to use clocksource clock.
 *
 * @tk:                The target timekeeper to setup.
 * @clock:                Pointer to clocksource.
 *
 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
 * pair and interval request.
 *
 * Unless you're the timekeeping code, you should not be using this!
 */
static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
{
        u64 interval;
        struct clocksource *old_clock;

        ++tk->cs_was_changed_seq;
        old_clock = tk->tkr_mono.clock;
        tk->tkr_mono.clock = clock;
        tk->tkr_mono.mask = clock->mask;
        tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);

        tk->tkr_raw.clock = clock;
        tk->tkr_raw.mask = clock->mask;
        tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;

        /* Do the ns -> cycle conversion first, using original mult */
        interval = (u64)NTP_INTERVAL_LENGTH << clock->shift;
        interval += clock->mult / 2;
        do_div(interval, clock->mult);
        if (interval == 0)
                interval = 1;

        tk->cycle_interval = interval;

        /* Go back from cycles -> shifted ns */
        tk->xtime_interval = interval * clock->mult;
        tk->raw_interval = interval * clock->mult;

         /* if changing clocks, convert xtime_nsec shift units */
        if (old_clock) {
                int shift_change = clock->shift - old_clock->shift;
                if (shift_change < 0) {
                        tk->tkr_mono.xtime_nsec >>= -shift_change;
                        tk->tkr_raw.xtime_nsec >>= -shift_change;
                } else {
                        tk->tkr_mono.xtime_nsec <<= shift_change;
                        tk->tkr_raw.xtime_nsec <<= shift_change;
                }
        }

        tk->tkr_mono.shift = clock->shift;
        tk->tkr_raw.shift = clock->shift;

        tk->ntp_error = 0;
        tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;

        /*
         * ntp_tick is the tick length that NTP disciplines (its ±500 PPM
         * scales only this part), in NTP-shifted ns: the real interval of
         * a whole number of counter cycles. Because cycle_interval is
         * rounded to an integer number of cycles, this ntp_tick differs
         * from the true intended 1/HZ tick length by up to half a cycle
         * period.
         */
        tk->ntp_tick = (u64)tk->xtime_interval << tk->ntp_error_shift;

        /*
         * cs_tick_adj is the constant difference between the disciplined
         * ntp_tick above and the true 1/HZ tick, expressed per-second to
         * match the ntp_update_frequency() addends and handed to NTP via
         * ntp_clear() to be explicitly included in its tick_length.
         *
         * Worked example: HZ=1000, ACPI PM timer at 3.579545 MHz, which
         * has 3579.545 cycles in 1ms, rounded to cycle_interval = 3580.
         *
         * So ntp_tick is actually 1.000127ms, as that is the amount of
         * time that 3580 cycles will take at the nominal frequency. This
         * is the part that NTP disciplines, causing each 3580 counts to
         * advance the clock by up to NTP's ±500PPM of that amount.
         *
         * The "extra" 127ns/tick is what's stored in cs_tick_adj and
         * applied as a constant correction by ntp_update_frequency() so
         * that NTP *believes* it's disciplining a 1ms tick.
         */
        tk->cs_tick_adj = (s64)tk->ntp_tick -
                          ((s64)NTP_INTERVAL_LENGTH << NTP_SCALE_SHIFT);
        tk->cs_tick_adj *= NTP_INTERVAL_FREQ;

        /*
         * The timekeeper keeps its own mult values for the currently
         * active clocksource. These value will be adjusted via NTP
         * to counteract clock drifting.
         */
        tk->tkr_mono.mult = clock->mult;
        tk->tkr_raw.mult = clock->mult;
        tk->ntp_err_mult = 0;
        tk->skip_second_overflow = 0;
        tk->skew_delta = 0;

        tk->cs_id = clock->id;

        /* Coupled clockevent data */
        if (IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED) &&
            clock->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT) {
                /*
                 * Aim for an one hour maximum delta and use KHz to handle
                 * clocksources with a frequency above 4GHz correctly as
                 * the frequency argument of clocks_calc_mult_shift() is u32.
                 */
                clocks_calc_mult_shift(&tk->cs_ns_to_cyc_mult, &tk->cs_ns_to_cyc_shift,
                                       NSEC_PER_MSEC, clock->freq_khz, 3600 * 1000);
                /*
                 * Initialize the conversion limit as the previous clocksource
                 * might have the same shift/mult pair so the quick check in
                 * tk_update_ns_to_cyc() fails to update it after a clocksource
                 * change leaving it effectivly zero.
                 */
                tk->cs_ns_to_cyc_maxns = div_u64(clock->mask, tk->cs_ns_to_cyc_mult);
        }
}

/* Timekeeper helper functions. */
static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta)
{
        return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift);
}

static __always_inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
{
        /* Calculate the delta since the last update_wall_time() */
        u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask;

        /*
         * This detects both negative motion and the case where the delta
         * overflows the multiplication with tkr->mult.
         */
        if (unlikely(delta > tkr->clock->max_cycles)) {
                /*
                 * Handle clocksource inconsistency between CPUs to prevent
                 * time from going backwards by checking for the MSB of the
                 * mask being set in the delta.
                 */
                if (delta & ~(mask >> 1))
                        return tkr->xtime_nsec >> tkr->shift;

                return delta_to_ns_safe(tkr, delta);
        }

        return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift;
}

static __always_inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
{
        return timekeeping_cycles_to_ns(tkr, tk_clock_read(tkr));
}

/**
 * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
 * @tkr: Timekeeping readout base from which we take the update
 * @tkf: Pointer to NMI safe timekeeper
 *
 * We want to use this from any context including NMI and tracing /
 * instrumenting the timekeeping code itself.
 *
 * Employ the latch technique; see @write_seqcount_latch.
 *
 * So if a NMI hits the update of base[0] then it will use base[1]
 * which is still consistent. In the worst case this can result is a
 * slightly wrong timestamp (a few nanoseconds). See
 * @ktime_get_mono_fast_ns.
 */
static void update_fast_timekeeper(const struct tk_read_base *tkr,
                                   struct tk_fast *tkf)
{
        struct tk_read_base *base = tkf->base;

        /* Force readers off to base[1] */
        write_seqcount_latch_begin(&tkf->seq);

        /* Update base[0] */
        memcpy(base, tkr, sizeof(*base));

        /* Force readers back to base[0] */
        write_seqcount_latch(&tkf->seq);

        /* Update base[1] */
        memcpy(base + 1, base, sizeof(*base));

        write_seqcount_latch_end(&tkf->seq);
}

static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
{
        struct tk_read_base *tkr;
        unsigned int seq;
        u64 now;

        do {
                seq = read_seqcount_latch(&tkf->seq);
                tkr = tkf->base + (seq & 0x01);
                now = ktime_to_ns(tkr->base);
                now += timekeeping_get_ns(tkr);
        } while (read_seqcount_latch_retry(&tkf->seq, seq));

        return now;
}

/**
 * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
 *
 * This timestamp is not guaranteed to be monotonic across an update.
 * The timestamp is calculated by:
 *
 *        now = base_mono + clock_delta * slope
 *
 * So if the update lowers the slope, readers who are forced to the
 * not yet updated second array are still using the old steeper slope.
 *
 * tmono
 * ^
 * |    o  n
 * |   o n
 * |  u
 * | o
 * |o
 * |12345678---> reader order
 *
 * o = old slope
 * u = update
 * n = new slope
 *
 * So reader 6 will observe time going backwards versus reader 5.
 *
 * While other CPUs are likely to be able to observe that, the only way
 * for a CPU local observation is when an NMI hits in the middle of
 * the update. Timestamps taken from that NMI context might be ahead
 * of the following timestamps. Callers need to be aware of that and
 * deal with it.
 */
u64 notrace ktime_get_mono_fast_ns(void)
{
        return __ktime_get_fast_ns(&tk_fast_mono);
}
EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);

/**
 * ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw
 *
 * Contrary to ktime_get_mono_fast_ns() this is always correct because the
 * conversion factor is not affected by NTP/PTP correction.
 */
u64 notrace ktime_get_raw_fast_ns(void)
{
        return __ktime_get_fast_ns(&tk_fast_raw);
}
EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);

/**
 * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
 *
 * To keep it NMI safe since we're accessing from tracing, we're not using a
 * separate timekeeper with updates to monotonic clock and boot offset
 * protected with seqcounts. This has the following minor side effects:
 *
 * (1) Its possible that a timestamp be taken after the boot offset is updated
 * but before the timekeeper is updated. If this happens, the new boot offset
 * is added to the old timekeeping making the clock appear to update slightly
 * earlier:
 *    CPU 0                                        CPU 1
 *    timekeeping_inject_sleeptime64()
 *    __timekeeping_inject_sleeptime(tk, delta);
 *                                                 timestamp();
 *    timekeeping_update_staged(tkd, TK_CLEAR_NTP...);
 *
 * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
 * partially updated.  Since the tk->offs_boot update is a rare event, this
 * should be a rare occurrence which postprocessing should be able to handle.
 *
 * The caveats vs. timestamp ordering as documented for ktime_get_mono_fast_ns()
 * apply as well.
 */
u64 notrace ktime_get_boot_fast_ns(void)
{
        struct timekeeper *tk = &tk_core.timekeeper;

        return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot)));
}
EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);

/**
 * ktime_get_tai_fast_ns - NMI safe and fast access to tai clock.
 *
 * The same limitations as described for ktime_get_boot_fast_ns() apply. The
 * mono time and the TAI offset are not read atomically which may yield wrong
 * readouts. However, an update of the TAI offset is an rare event e.g., caused
 * by settime or adjtimex with an offset. The user of this function has to deal
 * with the possibility of wrong timestamps in post processing.
 */
u64 notrace ktime_get_tai_fast_ns(void)
{
        struct timekeeper *tk = &tk_core.timekeeper;

        return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai)));
}
EXPORT_SYMBOL_GPL(ktime_get_tai_fast_ns);

/**
 * ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime.
 *
 * See ktime_get_mono_fast_ns() for documentation of the time stamp ordering.
 */
u64 ktime_get_real_fast_ns(void)
{
        struct tk_fast *tkf = &tk_fast_mono;
        struct tk_read_base *tkr;
        u64 baser, delta;
        unsigned int seq;

        do {
                seq = raw_read_seqcount_latch(&tkf->seq);
                tkr = tkf->base + (seq & 0x01);
                baser = ktime_to_ns(tkr->base_real);
                delta = timekeeping_get_ns(tkr);
        } while (raw_read_seqcount_latch_retry(&tkf->seq, seq));

        return baser + delta;
}
EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns);

/**
 * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
 * @tk: Timekeeper to snapshot.
 *
 * It generally is unsafe to access the clocksource after timekeeping has been
 * suspended, so take a snapshot of the readout base of @tk and use it as the
 * fast timekeeper's readout base while suspended.  It will return the same
 * number of cycles every time until timekeeping is resumed at which time the
 * proper readout base for the fast timekeeper will be restored automatically.
 */
static void halt_fast_timekeeper(const struct timekeeper *tk)
{
        static struct tk_read_base tkr_dummy;
        const struct tk_read_base *tkr = &tk->tkr_mono;

        memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
        cycles_at_suspend = tk_clock_read(tkr);
        tkr_dummy.clock = &dummy_clock;
        tkr_dummy.base_real = tkr->base + tk->offs_real;
        update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);

        tkr = &tk->tkr_raw;
        memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
        tkr_dummy.clock = &dummy_clock;
        update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
}

static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);

static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
{
        raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
}

/**
 * pvclock_gtod_register_notifier - register a pvclock timedata update listener
 * @nb: Pointer to the notifier block to register
 */
int pvclock_gtod_register_notifier(struct notifier_block *nb)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        int ret;

        guard(raw_spinlock_irqsave)(&tk_core.lock);
        ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
        update_pvclock_gtod(tk, true);

        return ret;
}
EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);

/**
 * pvclock_gtod_unregister_notifier - unregister a pvclock
 * timedata update listener
 * @nb: Pointer to the notifier block to unregister
 */
int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
{
        guard(raw_spinlock_irqsave)(&tk_core.lock);
        return raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
}
EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);

/*
 * tk_update_leap_state - helper to update the next_leap_ktime
 */
static inline void tk_update_leap_state(struct timekeeper *tk)
{
        tk->next_leap_ktime = ntp_get_next_leap(tk->id);
        if (tk->next_leap_ktime != KTIME_MAX)
                /* Convert to monotonic time */
                tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
}

/*
 * Leap state update for both shadow and the real timekeeper
 * Separate to spare a full memcpy() of the timekeeper.
 */
static void tk_update_leap_state_all(struct tk_data *tkd)
{
        write_seqcount_begin(&tkd->seq);
        tk_update_leap_state(&tkd->shadow_timekeeper);
        tkd->timekeeper.next_leap_ktime = tkd->shadow_timekeeper.next_leap_ktime;
        write_seqcount_end(&tkd->seq);
}

/*
 * Update the ktime_t based scalar nsec members of the timekeeper
 */
static inline void tk_update_ktime_data(struct timekeeper *tk)
{
        u64 seconds;
        u32 nsec;

        /*
         * The xtime based monotonic readout is:
         *        nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
         * The ktime based monotonic readout is:
         *        nsec = base_mono + now();
         * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
         */
        seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
        nsec = (u32) tk->wall_to_monotonic.tv_nsec;
        tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);

        /*
         * The sum of the nanoseconds portions of xtime and
         * wall_to_monotonic can be greater/equal one second. Take
         * this into account before updating tk->ktime_sec.
         */
        nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
        if (nsec >= NSEC_PER_SEC)
                seconds++;
        tk->ktime_sec = seconds;

        /* Update the monotonic raw base */
        tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
}

static inline void tk_update_ns_to_cyc(struct timekeeper *tks, struct timekeeper *tkc)
{
        struct tk_read_base *tkrs = &tks->tkr_mono;
        struct tk_read_base *tkrc = &tkc->tkr_mono;
        unsigned int shift;

        if (!IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED) ||
            !(tkrs->clock->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT))
                return;

        if (tkrs->mult == tkrc->mult && tkrs->shift == tkrc->shift)
                return;
        /*
         * The conversion math is simple:
         *
         *      CS::MULT       (1 << NS_TO_CYC_SHIFT)
         *   --------------- = ----------------------
         *   (1 << CS:SHIFT)       NS_TO_CYC_MULT
         *
         * Ergo:
         *
         *   NS_TO_CYC_MULT = (1 << (CS::SHIFT + NS_TO_CYC_SHIFT)) / CS::MULT
         *
         * NS_TO_CYC_SHIFT has been set up in tk_setup_internals()
         */
        shift = tkrs->shift + tks->cs_ns_to_cyc_shift;
        tks->cs_ns_to_cyc_mult = (u32)div_u64(1ULL << shift, tkrs->mult);
        tks->cs_ns_to_cyc_maxns = div_u64(tkrs->clock->mask, tks->cs_ns_to_cyc_mult);
}

/*
 * Restore the shadow timekeeper from the real timekeeper.
 */
static void timekeeping_restore_shadow(struct tk_data *tkd)
{
        lockdep_assert_held(&tkd->lock);
        memcpy(&tkd->shadow_timekeeper, &tkd->timekeeper, sizeof(tkd->timekeeper));
}

static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int action)
{
        struct timekeeper *tk = &tkd->shadow_timekeeper;

        lockdep_assert_held(&tkd->lock);

        /*
         * Block out readers before running the updates below because that
         * updates VDSO and other time related infrastructure. Not blocking
         * the readers might let a reader see time going backwards when
         * reading from the VDSO after the VDSO update and then reading in
         * the kernel from the timekeeper before that got updated.
         */
        write_seqcount_begin(&tkd->seq);

        if (action & TK_CLEAR_NTP) {
                tk->ntp_error = 0;
                ntp_clear(tk->id, tk->cs_tick_adj);
        }

        tk_update_leap_state(tk);
        tk_update_ktime_data(tk);
        tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;

        if (tk->id == TIMEKEEPER_CORE) {
                tk_update_ns_to_cyc(tk, &tkd->timekeeper);
                update_vsyscall(tk);
                update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);

                update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
                update_fast_timekeeper(&tk->tkr_raw,  &tk_fast_raw);
        } else if (tk_is_aux(tk)) {
                vdso_time_update_aux(tk);
        }

        if (action & TK_CLOCK_WAS_SET)
                tk->clock_was_set_seq++;

        /*
         * Update the real timekeeper.
         *
         * We could avoid this memcpy() by switching pointers, but that has
         * the downside that the reader side does not longer benefit from
         * the cacheline optimized data layout of the timekeeper and requires
         * another indirection.
         *
         * Write xtime_sec first so that even if the memcpy() tears the store
         * data integrity is provided for ktime_get_real_seconds().
         */
        WRITE_ONCE(tkd->timekeeper.xtime_sec, tk->xtime_sec);
        memcpy(&tkd->timekeeper, tk, sizeof(*tk));
        write_seqcount_end(&tkd->seq);
}

/**
 * timekeeping_forward_now - update clock to the current time
 * @tk:                Pointer to the timekeeper to update
 *
 * Forward the current clock to update its state since the last call to
 * update_wall_time(). This is useful before significant clock changes,
 * as it avoids having to deal with this time offset explicitly.
 */
static void timekeeping_forward_now(struct timekeeper *tk)
{
        u64 cycle_now, delta;

        cycle_now = tk_clock_read(&tk->tkr_mono);
        delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask,
                                  tk->tkr_mono.clock->max_raw_delta);
        tk->tkr_mono.cycle_last = cycle_now;
        tk->tkr_raw.cycle_last  = cycle_now;

        while (delta > 0) {
                u64 max = tk->tkr_mono.clock->max_cycles;
                u64 incr = delta < max ? delta : max;

                tk->tkr_mono.xtime_nsec += incr * tk->tkr_mono.mult;
                tk->tkr_raw.xtime_nsec += incr * tk->tkr_raw.mult;
                tk_normalize_xtime(tk);
                delta -= incr;
        }
        tk_update_coarse_nsecs(tk);
}

/*
 * ktime_expiry_to_cycles - Convert a expiry time to clocksource cycles
 * @id:                Clocksource ID which is required for validity
 * @expires_ns:        Absolute CLOCK_MONOTONIC expiry time (nsecs) to be converted
 * @cycles:        Pointer to storage for corresponding absolute cycles value
 *
 * Convert a CLOCK_MONOTONIC based absolute expiry time to a cycles value
 * based on the correlated clocksource of the clockevent device by using
 * the base nanoseconds and cycles values of the last timekeeper update and
 * converting the delta between @expires_ns and base nanoseconds to cycles.
 *
 * This only works for clockevent devices which are using a less than or
 * equal comparator against the clocksource.
 *
 * Utilizing this avoids two clocksource reads for such devices, the
 * ktime_get() in clockevents_program_event() to calculate the delta expiry
 * value and the readout in the device::set_next_event() callback to
 * convert the delta back to a absolute comparator value.
 *
 * Returns: True if @id matches the current clocksource ID, false otherwise
 */
bool ktime_expiry_to_cycles(enum clocksource_ids id, ktime_t expires_ns, u64 *cycles)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        struct tk_read_base *tkrm = &tk->tkr_mono;
        ktime_t base_ns, delta_ns, max_ns;
        u64 base_cycles, delta_cycles;
        unsigned int seq;
        u32 mult, shift;

        /*
         * Racy check to avoid the seqcount overhead when ID does not match. If
         * the relevant clocksource is installed concurrently, then this will
         * just delay the switch over to this mechanism until the next event is
         * programmed. If the ID is not matching the clock events code will use
         * the regular relative set_next_event() callback as before.
         */
        if (data_race(tk->cs_id) != id)
                return false;

        do {
                seq = read_seqcount_begin(&tk_core.seq);

                if (tk->cs_id != id)
                        return false;

                base_cycles = tkrm->cycle_last;
                base_ns = tkrm->base + (tkrm->xtime_nsec >> tkrm->shift);

                mult = tk->cs_ns_to_cyc_mult;
                shift = tk->cs_ns_to_cyc_shift;
                max_ns = tk->cs_ns_to_cyc_maxns;

        } while (read_seqcount_retry(&tk_core.seq, seq));

        /* Prevent negative deltas and multiplication overflows */
        delta_ns = min(expires_ns - base_ns, max_ns);
        delta_ns = max(delta_ns, 0);

        /* Convert to cycles */
        delta_cycles = ((u64)delta_ns * mult) >> shift;
        *cycles = base_cycles + delta_cycles;
        return true;
}

/**
 * ktime_get_real_ts64 - Returns the time of day in a timespec64.
 * @ts:                pointer to the timespec to be set
 *
 * Returns the time of day in a timespec64 (WARN if suspended).
 */
void ktime_get_real_ts64(struct timespec64 *ts)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        unsigned int seq;
        u64 nsecs;

        WARN_ON(timekeeping_suspended);

        do {
                seq = read_seqcount_begin(&tk_core.seq);

                ts->tv_sec = tk->xtime_sec;
                nsecs = timekeeping_get_ns(&tk->tkr_mono);

        } while (read_seqcount_retry(&tk_core.seq, seq));

        ts->tv_nsec = 0;
        timespec64_add_ns(ts, nsecs);
}
EXPORT_SYMBOL(ktime_get_real_ts64);

ktime_t ktime_get(void)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        unsigned int seq;
        ktime_t base;
        u64 nsecs;

        WARN_ON(timekeeping_suspended);

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                base = tk->tkr_mono.base;
                nsecs = timekeeping_get_ns(&tk->tkr_mono);

        } while (read_seqcount_retry(&tk_core.seq, seq));

        return ktime_add_ns(base, nsecs);
}
EXPORT_SYMBOL_GPL(ktime_get);

u32 ktime_get_resolution_ns(void)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        unsigned int seq;
        u32 nsecs;

        WARN_ON(timekeeping_suspended);

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
        } while (read_seqcount_retry(&tk_core.seq, seq));

        return nsecs;
}
EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);

static const ktime_t *const offsets[TK_OFFS_MAX] = {
        [TK_OFFS_REAL]        = &tk_core.timekeeper.offs_real,
        [TK_OFFS_BOOT]        = &tk_core.timekeeper.offs_boot,
        [TK_OFFS_TAI]        = &tk_core.timekeeper.offs_tai,
};

ktime_t ktime_get_with_offset(enum tk_offsets offs)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        const ktime_t *offset = offsets[offs];
        unsigned int seq;
        ktime_t base;
        u64 nsecs;

        WARN_ON(timekeeping_suspended);

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                base = ktime_add(tk->tkr_mono.base, *offset);
                nsecs = timekeeping_get_ns(&tk->tkr_mono);

        } while (read_seqcount_retry(&tk_core.seq, seq));

        return ktime_add_ns(base, nsecs);

}
EXPORT_SYMBOL_GPL(ktime_get_with_offset);

ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        const ktime_t *offset = offsets[offs];
        unsigned int seq;
        ktime_t base;
        u64 nsecs;

        WARN_ON(timekeeping_suspended);

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                base = ktime_add(tk->tkr_mono.base, *offset);
                nsecs = tk->coarse_nsec;

        } while (read_seqcount_retry(&tk_core.seq, seq));

        return ktime_add_ns(base, nsecs);
}
EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);

/**
 * ktime_mono_to_any() - convert monotonic time to any other time
 * @tmono:        time to convert.
 * @offs:        which offset to use
 */
ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
{
        const ktime_t *offset = offsets[offs];
        unsigned int seq;
        ktime_t tconv;

        if (IS_ENABLED(CONFIG_64BIT)) {
                /*
                 * Paired with WRITE_ONCE()s in tk_set_wall_to_mono() and
                 * tk_update_sleep_time().
                 */
                return ktime_add(tmono, READ_ONCE(*offset));
        }

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                tconv = ktime_add(tmono, *offset);
        } while (read_seqcount_retry(&tk_core.seq, seq));

        return tconv;
}
EXPORT_SYMBOL_GPL(ktime_mono_to_any);

/**
 * ktime_get_raw - Returns the raw monotonic time in ktime_t format
 */
ktime_t ktime_get_raw(void)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        unsigned int seq;
        ktime_t base;
        u64 nsecs;

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                base = tk->tkr_raw.base;
                nsecs = timekeeping_get_ns(&tk->tkr_raw);

        } while (read_seqcount_retry(&tk_core.seq, seq));

        return ktime_add_ns(base, nsecs);
}
EXPORT_SYMBOL_GPL(ktime_get_raw);

/**
 * ktime_get_ts64 - get the monotonic clock in timespec64 format
 * @ts:                pointer to timespec variable
 *
 * The function calculates the monotonic clock from the realtime
 * clock and the wall_to_monotonic offset and stores the result
 * in normalized timespec64 format in the variable pointed to by @ts.
 */
void ktime_get_ts64(struct timespec64 *ts)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        struct timespec64 tomono;
        unsigned int seq;
        u64 nsec;

        WARN_ON(timekeeping_suspended);

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                ts->tv_sec = tk->xtime_sec;
                nsec = timekeeping_get_ns(&tk->tkr_mono);
                tomono = tk->wall_to_monotonic;

        } while (read_seqcount_retry(&tk_core.seq, seq));

        ts->tv_sec += tomono.tv_sec;
        ts->tv_nsec = 0;
        timespec64_add_ns(ts, nsec + tomono.tv_nsec);
}
EXPORT_SYMBOL_GPL(ktime_get_ts64);

/**
 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
 *
 * Returns the seconds portion of CLOCK_MONOTONIC with a single non
 * serialized read. tk->ktime_sec is of type 'unsigned long' so this
 * works on both 32 and 64 bit systems. On 32 bit systems the readout
 * covers ~136 years of uptime which should be enough to prevent
 * premature wrap arounds.
 */
time64_t ktime_get_seconds(void)
{
        struct timekeeper *tk = &tk_core.timekeeper;

        WARN_ON(timekeeping_suspended);
        return tk->ktime_sec;
}
EXPORT_SYMBOL_GPL(ktime_get_seconds);

/**
 * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
 *
 * Returns the wall clock seconds since 1970.
 *
 * For 64bit systems the fast access to tk->xtime_sec is preserved. On
 * 32bit systems the access must be protected with the sequence
 * counter to provide "atomic" access to the 64bit tk->xtime_sec
 * value.
 */
time64_t ktime_get_real_seconds(void)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        time64_t seconds;
        unsigned int seq;

        if (IS_ENABLED(CONFIG_64BIT))
                return READ_ONCE(tk->xtime_sec);

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                seconds = READ_ONCE(tk->xtime_sec);

        } while (read_seqcount_retry(&tk_core.seq, seq));

        return seconds;
}
EXPORT_SYMBOL_GPL(ktime_get_real_seconds);

/**
 * __ktime_get_real_seconds - Unprotected access to CLOCK_REALTIME seconds
 *
 * The same as ktime_get_real_seconds() but without the sequence counter
 * protection. This function is used in restricted contexts like the x86 MCE
 * handler and in KGDB. It's unprotected on 32-bit vs. concurrent half
 * completed modification and only to be used for such critical contexts.
 *
 * Returns: Racy snapshot of the CLOCK_REALTIME seconds value
 */
noinstr time64_t __ktime_get_real_seconds(void)
{
        struct timekeeper *tk = &tk_core.timekeeper;

        return READ_ONCE(tk->xtime_sec);
}

static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr,
                                         struct clocksource_hw_snapshot *chs)
{
        struct clocksource *clock = READ_ONCE(tkr->clock);

        if (unlikely(clock->read_snapshot))
                return clock->read_snapshot(clock, chs);

        return clock->read(clock);
}


/**
 * ktime_get_snapshot_id -  Simultaneously snapshot a given clock ID with
 *                            the corresponding monotonic raw and the underlying
 *                            clocksource counter value.
 * @clock_id:                The clock ID to snapshot
 * @systime_snapshot:        Pointer to struct receiving the system time snapshot
 *
 * For the system time keeping clocks (REALTIME, MONOTONIC and BOOTTIME) the
 * monotonic raw clock is CLOCK_MONOTONIC_RAW. For AUX clocks this is the
 * monotonic raw clock related to the AUX clock. These AUX clock related
 * monotonic raw clocks have a strict linear offset to the system time
 * CLOCK_MONOTONIC_RAW:
 *
 *        MONOTONIC_RAW(AUX$N) = CLOCK_MONOTONIC_RAW(system) + offset(AUX$N)
 *
 * The offset is established when a AUX clock is initialized, but it is
 * currently not accessible.
 */
void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot)
{
        ktime_t base_raw, base_sys, offs_sys, *offs, offs_zero = 0;
        u64 nsec_raw, nsec_sys, now;
        struct timekeeper *tk;
        struct tk_data *tkd;
        unsigned int seq;

        /* Invalidate the snapshot for all failure cases */
        systime_snapshot->valid = false;

        if (WARN_ON_ONCE(timekeeping_suspended))
                return;

        switch (clock_id) {
        case CLOCK_REALTIME:
                tkd = &tk_core;
                offs = &tk_core.timekeeper.offs_real;
                break;
        /* Map RAW to MONOTONIC so the loop below is trivial */
        case CLOCK_MONOTONIC_RAW:
        case CLOCK_MONOTONIC:
                tkd = &tk_core;
                offs = &offs_zero;
                break;
        case CLOCK_BOOTTIME:
                tkd = &tk_core;
                offs = &tk_core.timekeeper.offs_boot;
                break;
        case CLOCK_AUX ... CLOCK_AUX_LAST:
                tkd = aux_get_tk_data(clock_id);
                if (!tkd)
                        return;
                offs = &tkd->timekeeper.offs_aux;
                break;
        default:
                WARN_ON_ONCE(1);
                return;
        }

        tk = &tkd->timekeeper;

        do {
                struct clocksource_hw_snapshot chs = { };

                seq = read_seqcount_begin(&tkd->seq);

                /* Aux clocks can be invalid */
                if (!tk->clock_valid)
                        return;

                now = tk_clock_read_snapshot(&tk->tkr_mono, &chs);
                systime_snapshot->cs_id = tk->tkr_mono.clock->id;

                systime_snapshot->hw_cycles = chs.hw_cycles;
                systime_snapshot->hw_csid = chs.hw_csid;

                systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
                systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;

                base_sys = tk->tkr_mono.base;
                offs_sys = *offs;
                base_raw = tk->tkr_raw.base;

                nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
                nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
        } while (read_seqcount_retry(&tkd->seq, seq));

        systime_snapshot->cycles = now;
        systime_snapshot->systime = ktime_add_ns(base_sys, offs_sys + nsec_sys);
        systime_snapshot->monoraw = ktime_add_ns(base_raw, nsec_raw);

        /*
         * Special case for PTP. Just transfer the raw time into sys,
         * so the call sites can consistently use snap::systime.
         */
        if (clock_id == CLOCK_MONOTONIC_RAW)
                systime_snapshot->systime = systime_snapshot->monoraw;
        /* Tell the consumer that this snapshot is valid */
        systime_snapshot->valid = true;
}
EXPORT_SYMBOL_GPL(ktime_get_snapshot_id);

/* Scale base by mult/div checking for overflow */
static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
{
        u64 tmp, rem;

        tmp = div64_u64_rem(*base, div, &rem);

        if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
            ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
                return -EOVERFLOW;
        tmp *= mult;

        rem = div64_u64(rem * mult, div);
        *base = tmp + rem;
        return 0;
}

/**
 * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
 * @history:                        Snapshot representing start of history
 * @partial_history_cycles:        Cycle offset into history (fractional part)
 * @total_history_cycles:        Total history length in cycles
 * @discontinuity:                True indicates clock was set on history period
 * @ts:                                Cross timestamp that should be adjusted using
 *        partial/total ratio
 *
 * Helper function used by get_device_system_crosststamp() to correct the
 * crosstimestamp corresponding to the start of the current interval to the
 * system counter value (timestamp point) provided by the driver. The
 * total_history_* quantities are the total history starting at the provided
 * reference point and ending at the start of the current interval. The cycle
 * count between the driver timestamp point and the start of the current
 * interval is partial_history_cycles.
 */
static int adjust_historical_crosststamp(struct system_time_snapshot *history,
                                         u64 partial_history_cycles,
                                         u64 total_history_cycles,
                                         bool discontinuity,
                                         struct system_device_crosststamp *ts)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        u64 corr_raw, corr_sys;
        bool interp_forward;
        int ret;

        if (total_history_cycles == 0 || partial_history_cycles == 0)
                return 0;

        /* Interpolate shortest distance from beginning or end of history */
        interp_forward = partial_history_cycles > total_history_cycles / 2;
        partial_history_cycles = interp_forward ?
                total_history_cycles - partial_history_cycles :
                partial_history_cycles;

        /*
         * Scale the monotonic raw time delta by:
         *        partial_history_cycles / total_history_cycles
         */
        corr_raw = (u64)ktime_to_ns(ktime_sub(ts->sys_monoraw, history->monoraw));
        ret = scale64_check_overflow(partial_history_cycles,
                                     total_history_cycles, &corr_raw);
        if (ret)
                return ret;

        /*
         * If there is a discontinuity in the history, scale monotonic raw
         * correction by:
         *        mult(sys)/mult(raw) yielding the system time correction
         *
         * Otherwise, calculate the system time correction similar to monotonic
         * raw calculation
         */
        if (discontinuity) {
                corr_sys = mul_u64_u32_div(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
        } else {
                corr_sys = (u64)ktime_to_ns(ktime_sub(ts->sys_systime, history->systime));
                ret = scale64_check_overflow(partial_history_cycles, total_history_cycles,
                                             &corr_sys);
                if (ret)
                        return ret;
        }

        /* Fixup monotonic raw and system time time values */
        if (interp_forward) {
                ts->sys_monoraw = ktime_add_ns(history->monoraw, corr_raw);
                ts->sys_systime = ktime_add_ns(history->systime, corr_sys);
        } else {
                ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
                ts->sys_systime = ktime_sub_ns(ts->sys_systime, corr_sys);
        }

        return 0;
}

/*
 * timestamp_in_interval - true if ts is chronologically in [start, end]
 *
 * True if ts occurs chronologically at or after start, and before or at end.
 */
static bool timestamp_in_interval(u64 start, u64 end, u64 ts)
{
        if (ts >= start && ts <= end)
                return true;
        if (start > end && (ts >= start || ts <= end))
                return true;
        return false;
}

static bool convert_clock(u64 *val, u32 numerator, u32 denominator)
{
        u64 rem, res;

        if (!numerator || !denominator)
                return false;

        res = div64_u64_rem(*val, denominator, &rem) * numerator;
        *val = res + div_u64(rem * numerator, denominator);
        return true;
}

static bool convert_base_to_cs(struct system_counterval_t *scv)
{
        struct clocksource *cs = tk_core.timekeeper.tkr_mono.clock;
        struct clocksource_base *base;
        u32 num, den;

        /* The timestamp was taken from the time keeper clock source */
        if (cs->id == scv->cs_id)
                return true;

        /*
         * Check whether cs_id matches the base clock. Prevent the compiler from
         * re-evaluating @base as the clocksource might change concurrently.
         */
        base = READ_ONCE(cs->base);
        if (!base || base->id != scv->cs_id)
                return false;

        num = scv->use_nsecs ? cs->freq_khz : base->numerator;
        den = scv->use_nsecs ? USEC_PER_SEC : base->denominator;

        if (!convert_clock(&scv->cycles, num, den))
                return false;

        scv->cycles += base->offset;
        /* Set the clocksource ID as scv::cycles is now clocksource based */
        scv->cs_id = cs->id;
        return true;
}

static bool convert_cs_to_base(u64 *cycles, enum clocksource_ids base_id)
{
        struct clocksource *cs = tk_core.timekeeper.tkr_mono.clock;
        struct clocksource_base *base;

        /*
         * Check whether base_id matches the base clock. Prevent the compiler from
         * re-evaluating @base as the clocksource might change concurrently.
         */
        base = READ_ONCE(cs->base);
        if (!base || base->id != base_id)
                return false;

        *cycles -= base->offset;
        if (!convert_clock(cycles, base->denominator, base->numerator))
                return false;
        return true;
}

static bool convert_ns_to_cs(u64 *delta)
{
        struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;

        if (BITS_TO_BYTES(fls64(*delta) + tkr->shift) >= sizeof(*delta))
                return false;

        *delta = div_u64((*delta << tkr->shift) - tkr->xtime_nsec, tkr->mult);
        return true;
}

/**
 * ktime_real_to_base_clock() - Convert CLOCK_REALTIME timestamp to a base clock timestamp
 * @treal:        CLOCK_REALTIME timestamp to convert
 * @base_id:        base clocksource id
 * @cycles:        pointer to store the converted base clock timestamp
 *
 * Converts a supplied, future realtime clock value to the corresponding base clock value.
 *
 * Return:  true if the conversion is successful, false otherwise.
 */
bool ktime_real_to_base_clock(ktime_t treal, enum clocksource_ids base_id, u64 *cycles)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        unsigned int seq;
        u64 delta;

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                if ((u64)treal < tk->tkr_mono.base_real)
                        return false;
                delta = (u64)treal - tk->tkr_mono.base_real;
                if (!convert_ns_to_cs(&delta))
                        return false;
                *cycles = tk->tkr_mono.cycle_last + delta;
                if (!convert_cs_to_base(cycles, base_id))
                        return false;
        } while (read_seqcount_retry(&tk_core.seq, seq));

        return true;
}
EXPORT_SYMBOL_GPL(ktime_real_to_base_clock);

/**
 * get_device_system_crosststamp - Synchronously capture system/device timestamp
 * @get_time_fn:        Callback to get simultaneous device time and system counter
 *                        from the device driver
 * @ctx:                Context passed to get_time_fn()
 * @history_begin:        Historical reference point used to interpolate system time when
 *                        the counter value provided by the driver is before the current interval
 * @xtstamp:                Receives simultaneously captured system and device time
 *
 * Reads a timestamp from a device and correlates it to system time
 *
 * See documentation for ktime_get_snapshot_id() for information about the raw
 * monotonic time stamp which is used here.
 */
int get_device_system_crosststamp(int (*get_time_fn)
                                  (ktime_t *device_time,
                                   struct system_counterval_t *sys_counterval,
                                   void *ctx),
                                  void *ctx,
                                  struct system_time_snapshot *history_begin,
                                  struct system_device_crosststamp *xtstamp)
{
        u64 syscnt_cycles, cycles, now, interval_start;
        ktime_t base_sys, base_raw, *offs;
        u32 clock_was_set_seq = 0;
        u64 nsec_sys, nsec_raw;
        u8 cs_was_changed_seq;
        unsigned int seq;
        bool do_interp;
        struct timekeeper *tk;
        struct tk_data *tkd;
        int ret;

        switch (xtstamp->clock_id) {
        case CLOCK_REALTIME:
                tkd = &tk_core;
                offs = &tk_core.timekeeper.offs_real;
                break;
        case CLOCK_AUX ... CLOCK_AUX_LAST:
                tkd = aux_get_tk_data(xtstamp->clock_id);
                if (!tkd)
                        return -ENODEV;
                offs = &tkd->timekeeper.offs_aux;
                break;
        default:
                WARN_ON_ONCE(1);
                return -ENODEV;
        }

        tk = &tkd->timekeeper;

        do {
                seq = read_seqcount_begin(&tkd->seq);
                /*
                 * Try to synchronously capture device time and a system
                 * counter value calling back into the device driver
                 */
                ret = get_time_fn(&xtstamp->device, &xtstamp->sys_counter, ctx);
                if (ret)
                        return ret;

                /*
                 * Verify that the clocksource ID associated with the captured
                 * system counter value is the same as for the currently
                 * installed timekeeper clocksource and convert to it.
                 */
                if (xtstamp->sys_counter.cs_id == CSID_GENERIC ||
                    !convert_base_to_cs(&xtstamp->sys_counter))
                        return -ENODEV;

                cycles = syscnt_cycles = xtstamp->sys_counter.cycles;

                /*
                 * Check whether the system counter value provided by the
                 * device driver is on the current timekeeping interval.
                 */
                now = tk_clock_read(&tk->tkr_mono);
                interval_start = tk->tkr_mono.cycle_last;
                if (!timestamp_in_interval(interval_start, now, cycles)) {
                        clock_was_set_seq = tk->clock_was_set_seq;
                        cs_was_changed_seq = tk->cs_was_changed_seq;
                        cycles = interval_start;
                        do_interp = true;
                } else {
                        do_interp = false;
                }

                base_sys = ktime_add(tk->tkr_mono.base, *offs);
                base_raw = tk->tkr_raw.base;

                nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles);
                nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles);
        } while (read_seqcount_retry(&tkd->seq, seq));

        xtstamp->sys_systime = ktime_add_ns(base_sys, nsec_sys);
        xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);

        /*
         * Interpolate if necessary, adjusting back from the start of the
         * current interval
         */
        if (do_interp) {
                u64 partial_history_cycles, total_history_cycles;
                bool discontinuity;

                /*
                 * Check that the counter value is not before the provided
                 * history reference and that the history doesn't cross a
                 * clocksource change
                 */
                if (!history_begin ||
                    !timestamp_in_interval(history_begin->cycles, cycles, syscnt_cycles) ||
                    history_begin->cs_was_changed_seq != cs_was_changed_seq)
                        return -EINVAL;

                partial_history_cycles = cycles - syscnt_cycles;
                total_history_cycles = cycles - history_begin->cycles;
                discontinuity = history_begin->clock_was_set_seq != clock_was_set_seq;

                ret = adjust_historical_crosststamp(history_begin, partial_history_cycles,
                                                    total_history_cycles, discontinuity, xtstamp);
        }

        return ret;
}
EXPORT_SYMBOL_GPL(get_device_system_crosststamp);

/**
 * timekeeping_clocksource_has_base - Check whether the current clocksource
 *                                      is based on given a base clock
 * @id:                base clocksource ID
 *
 * Note:        The return value is a snapshot which can become invalid right
 *                after the function returns.
 *
 * Return:        true if the timekeeper clocksource has a base clock with @id,
 *                false otherwise
 */
bool timekeeping_clocksource_has_base(enum clocksource_ids id)
{
        /*
         * This is a snapshot, so no point in using the sequence
         * count. Just prevent the compiler from re-evaluating @base as the
         * clocksource might change concurrently.
         */
        struct clocksource_base *base = READ_ONCE(tk_core.timekeeper.tkr_mono.clock->base);

        return base ? base->id == id : false;
}
EXPORT_SYMBOL_GPL(timekeeping_clocksource_has_base);

/**
 * do_settimeofday64 - Sets the time of day.
 * @ts:     pointer to the timespec64 variable containing the new time
 *
 * Sets the time of day to the new time and update NTP and notify hrtimers
 */
int do_settimeofday64(const struct timespec64 *ts)
{
        struct timespec64 ts_delta, xt;

        if (!timespec64_valid_settod(ts))
                return -EINVAL;

        scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
                struct timekeeper *tks = &tk_core.shadow_timekeeper;

                timekeeping_forward_now(tks);

                xt = tk_xtime(tks);
                ts_delta = timespec64_sub(*ts, xt);

                if (timespec64_compare(&tks->wall_to_monotonic, &ts_delta) > 0) {
                        timekeeping_restore_shadow(&tk_core);
                        return -EINVAL;
                }

                tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, ts_delta));
                tk_set_xtime(tks, ts);
                timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
        }

        /* Signal hrtimers about time change */
        clock_was_set(CLOCK_SET_WALL);

        audit_tk_injoffset(ts_delta);
        add_device_randomness(ts, sizeof(*ts));
        return 0;
}
EXPORT_SYMBOL(do_settimeofday64);

static inline bool timekeeper_is_core_tk(struct timekeeper *tk)
{
        return !IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS) || tk->id == TIMEKEEPER_CORE;
}

/**
 * __timekeeping_inject_offset - Adds or subtracts from the current time.
 * @tkd:        Pointer to the timekeeper to modify
 * @ts:                Pointer to the timespec variable containing the offset
 *
 * Adds or subtracts an offset value from the current time.
 */
static int __timekeeping_inject_offset(struct tk_data *tkd, const struct timespec64 *ts)
{
        struct timekeeper *tks = &tkd->shadow_timekeeper;
        struct timespec64 tmp;

        if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)
                return -EINVAL;

        timekeeping_forward_now(tks);

        if (timekeeper_is_core_tk(tks)) {
                /* Make sure the proposed value is valid */
                tmp = timespec64_add(tk_xtime(tks), *ts);
                if (timespec64_compare(&tks->wall_to_monotonic, ts) > 0 ||
                    !timespec64_valid_settod(&tmp)) {
                        timekeeping_restore_shadow(tkd);
                        return -EINVAL;
                }

                tk_xtime_add(tks, ts);
                tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, *ts));
        } else {
                struct tk_read_base *tkr_mono = &tks->tkr_mono;
                ktime_t now, offs;

                /* Get the current time */
                now = ktime_add_ns(tkr_mono->base, timekeeping_get_ns(tkr_mono));
                /* Add the relative offset change */
                offs = ktime_add(tks->offs_aux, timespec64_to_ktime(*ts));

                /* Prevent that the resulting time becomes negative */
                if (ktime_add(now, offs) < 0) {
                        timekeeping_restore_shadow(tkd);
                        return -EINVAL;
                }
                tk_update_aux_offs(tks, offs);
        }

        timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL);
        return 0;
}

static int timekeeping_inject_offset(const struct timespec64 *ts)
{
        int ret;

        scoped_guard (raw_spinlock_irqsave, &tk_core.lock)
                ret = __timekeeping_inject_offset(&tk_core, ts);

        /* Signal hrtimers about time change */
        if (!ret)
                clock_was_set(CLOCK_SET_WALL);
        return ret;
}

/*
 * Indicates if there is an offset between the system clock and the hardware
 * clock/persistent clock/rtc.
 */
int persistent_clock_is_local;

/*
 * Adjust the time obtained from the CMOS to be UTC time instead of
 * local time.
 *
 * This is ugly, but preferable to the alternatives.  Otherwise we
 * would either need to write a program to do it in /etc/rc (and risk
 * confusion if the program gets run more than once; it would also be
 * hard to make the program warp the clock precisely n hours)  or
 * compile in the timezone information into the kernel.  Bad, bad....
 *
 *                                                - TYT, 1992-01-01
 *
 * The best thing to do is to keep the CMOS clock in universal time (UTC)
 * as real UNIX machines always do it. This avoids all headaches about
 * daylight saving times and warping kernel clocks.
 */
void timekeeping_warp_clock(void)
{
        if (sys_tz.tz_minuteswest != 0) {
                struct timespec64 adjust;

                persistent_clock_is_local = 1;
                adjust.tv_sec = sys_tz.tz_minuteswest * 60;
                adjust.tv_nsec = 0;
                timekeeping_inject_offset(&adjust);
        }
}

/*
 * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
 */
static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
{
        tk->tai_offset = tai_offset;
        tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
}

/*
 * change_clocksource - Swaps clocksources if a new one is available
 *
 * Accumulates current time interval and initializes new clocksource
 */
static int change_clocksource(void *data)
{
        struct clocksource *new = data, *old = NULL;

        /*
         * If the clocksource is in a module, get a module reference.
         * Succeeds for built-in code (owner == NULL) as well. Abort if the
         * reference can't be acquired.
         */
        if (!try_module_get(new->owner))
                return 0;

        /* Abort if the device can't be enabled */
        if (new->enable && new->enable(new) != 0) {
                module_put(new->owner);
                return 0;
        }

        scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
                struct timekeeper *tks = &tk_core.shadow_timekeeper;

                timekeeping_forward_now(tks);
                old = tks->tkr_mono.clock;
                tk_setup_internals(tks, new);
                timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
        }

        tk_aux_update_clocksource();

        if (old) {
                if (old->disable)
                        old->disable(old);
                module_put(old->owner);
        }

        return 0;
}

/**
 * timekeeping_notify - Install a new clock source
 * @clock:                pointer to the clock source
 *
 * This function is called from clocksource.c after a new, better clock
 * source has been registered. The caller holds the clocksource_mutex.
 */
int timekeeping_notify(struct clocksource *clock)
{
        struct timekeeper *tk = &tk_core.timekeeper;

        if (tk->tkr_mono.clock == clock)
                return 0;

        /* Disable inlined reads accross the clocksource switch */
        clocksource_disable_inline_read();

        stop_machine(change_clocksource, clock, NULL);

        /*
         * If the clocksource has been selected and supports inlined reads
         * enable the branch.
         */
        if (tk->tkr_mono.clock == clock && clock->flags & CLOCK_SOURCE_CAN_INLINE_READ)
                clocksource_enable_inline_read();

        tick_clock_notify();
        return tk->tkr_mono.clock == clock ? 0 : -1;
}

/**
 * ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec
 * @ts:                pointer to the timespec64 to be set
 *
 * Returns the raw monotonic time (completely un-modified by ntp)
 */
void ktime_get_raw_ts64(struct timespec64 *ts)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        unsigned int seq;
        u64 nsecs;

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                ts->tv_sec = tk->raw_sec;
                nsecs = timekeeping_get_ns(&tk->tkr_raw);

        } while (read_seqcount_retry(&tk_core.seq, seq));

        ts->tv_nsec = 0;
        timespec64_add_ns(ts, nsecs);
}
EXPORT_SYMBOL(ktime_get_raw_ts64);

/**
 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
 */
int timekeeping_valid_for_hres(void)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        unsigned int seq;
        int ret;

        do {
                seq = read_seqcount_begin(&tk_core.seq);

                ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;

        } while (read_seqcount_retry(&tk_core.seq, seq));

        return ret;
}

/**
 * timekeeping_max_deferment - Returns max time the clocksource can be deferred
 */
u64 timekeeping_max_deferment(void)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        unsigned int seq;
        u64 ret;

        do {
                seq = read_seqcount_begin(&tk_core.seq);

                ret = tk->tkr_mono.clock->max_idle_ns;

        } while (read_seqcount_retry(&tk_core.seq, seq));

        return ret;
}

/**
 * read_persistent_clock64 -  Return time from the persistent clock.
 * @ts: Pointer to the storage for the readout value
 *
 * Weak dummy function for arches that do not yet support it.
 * Reads the time from the battery backed persistent clock.
 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
 *
 *  XXX - Do be sure to remove it once all arches implement it.
 */
void __weak read_persistent_clock64(struct timespec64 *ts)
{
        ts->tv_sec = 0;
        ts->tv_nsec = 0;
}

/**
 * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
 *                                        from the boot.
 * @wall_time:          current time as returned by persistent clock
 * @boot_offset:  offset that is defined as wall_time - boot_time
 *
 * Weak dummy function for arches that do not yet support it.
 *
 * The default function calculates offset based on the current value of
 * local_clock(). This way architectures that support sched_clock() but don't
 * support dedicated boot time clock will provide the best estimate of the
 * boot time.
 */
void __weak __init
read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
                                     struct timespec64 *boot_offset)
{
        read_persistent_clock64(wall_time);
        *boot_offset = ns_to_timespec64(local_clock());
}

static __init void tkd_basic_setup(struct tk_data *tkd, enum timekeeper_ids tk_id, bool valid)
{
        raw_spin_lock_init(&tkd->lock);
        seqcount_raw_spinlock_init(&tkd->seq, &tkd->lock);
        tkd->timekeeper.id = tkd->shadow_timekeeper.id = tk_id;
        tkd->timekeeper.clock_valid = tkd->shadow_timekeeper.clock_valid = valid;
}

/*
 * Flag reflecting whether timekeeping_resume() has injected sleeptime.
 *
 * The flag starts of false and is only set when a suspend reaches
 * timekeeping_suspend(), timekeeping_resume() sets it to false when the
 * timekeeper clocksource is not stopping across suspend and has been
 * used to update sleep time. If the timekeeper clocksource has stopped
 * then the flag stays true and is used by the RTC resume code to decide
 * whether sleeptime must be injected and if so the flag gets false then.
 *
 * If a suspend fails before reaching timekeeping_resume() then the flag
 * stays false and prevents erroneous sleeptime injection.
 */
static bool suspend_timing_needed;

/* Flag for if there is a persistent clock on this platform */
static bool persistent_clock_exists;

/*
 * timekeeping_init - Initializes the clocksource and common timekeeping values
 */
void __init timekeeping_init(void)
{
        struct timespec64 wall_time, boot_offset, wall_to_mono;
        struct timekeeper *tks = &tk_core.shadow_timekeeper;
        struct clocksource *clock;

        tkd_basic_setup(&tk_core, TIMEKEEPER_CORE, true);
        tk_aux_setup();

        read_persistent_wall_and_boot_offset(&wall_time, &boot_offset);
        if (timespec64_valid_settod(&wall_time) &&
            timespec64_to_ns(&wall_time) > 0) {
                persistent_clock_exists = true;
        } else if (timespec64_to_ns(&wall_time) != 0) {
                pr_warn("Persistent clock returned invalid value");
                wall_time = (struct timespec64){0};
        }

        if (timespec64_compare(&wall_time, &boot_offset) < 0)
                boot_offset = (struct timespec64){0};

        /*
         * We want set wall_to_mono, so the following is true:
         * wall time + wall_to_mono = boot time
         */
        wall_to_mono = timespec64_sub(boot_offset, wall_time);

        clock = clocksource_default_clock();
        if (clock->enable)
                clock->enable(clock);

        guard(raw_spinlock_irqsave)(&tk_core.lock);

        ntp_init();

        tk_setup_internals(tks, clock);

        tk_set_xtime(tks, &wall_time);
        tks->raw_sec = 0;

        tk_set_wall_to_mono(tks, wall_to_mono);

        /*
         * Use TK_UPDATE_ALL so the NTP layer picks up the clocksource's
         * cs_tick_adj via ntp_clear(). Clearing NTP here is otherwise
         * redundant as ntp_init() already initialised it above.
         */
        timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
}

/* time in seconds when suspend began for persistent clock */
static struct timespec64 timekeeping_suspend_time;

/**
 * __timekeeping_inject_sleeptime - Internal function to add sleep interval
 * @tk:                Pointer to the timekeeper to be updated
 * @delta:        Pointer to the delta value in timespec64 format
 *
 * Takes a timespec offset measuring a suspend interval and properly
 * adds the sleep offset to the timekeeping variables.
 */
static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
                                           const struct timespec64 *delta)
{
        if (!timespec64_valid_strict(delta)) {
                printk_deferred(KERN_WARNING
                                "__timekeeping_inject_sleeptime: Invalid "
                                "sleep delta value!\n");
                return;
        }
        tk_xtime_add(tk, delta);
        tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
        tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
        tk_debug_account_sleep_time(delta);
}

#if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
/*
 * We have three kinds of time sources to use for sleep time
 * injection, the preference order is:
 * 1) non-stop clocksource
 * 2) persistent clock (ie: RTC accessible when irqs are off)
 * 3) RTC
 *
 * 1) and 2) are used by timekeeping, 3) by RTC subsystem.
 * If system has neither 1) nor 2), 3) will be used finally.
 *
 *
 * If timekeeping has injected sleeptime via either 1) or 2),
 * 3) becomes needless, so in this case we don't need to call
 * rtc_resume(), and this is what timekeeping_rtc_skipresume()
 * means.
 */
bool timekeeping_rtc_skipresume(void)
{
        return !suspend_timing_needed;
}

/*
 * 1) can be determined whether to use or not only when doing
 * timekeeping_resume() which is invoked after rtc_suspend(),
 * so we can't skip rtc_suspend() surely if system has 1).
 *
 * But if system has 2), 2) will definitely be used, so in this
 * case we don't need to call rtc_suspend(), and this is what
 * timekeeping_rtc_skipsuspend() means.
 */
bool timekeeping_rtc_skipsuspend(void)
{
        return persistent_clock_exists;
}

/**
 * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
 * @delta: pointer to a timespec64 delta value
 *
 * This hook is for architectures that cannot support read_persistent_clock64
 * because their RTC/persistent clock is only accessible when irqs are enabled.
 * and also don't have an effective nonstop clocksource.
 *
 * This function should only be called by rtc_resume(), and allows
 * a suspend offset to be injected into the timekeeping values.
 */
void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
{
        scoped_guard(raw_spinlock_irqsave, &tk_core.lock) {
                struct timekeeper *tks = &tk_core.shadow_timekeeper;

                suspend_timing_needed = false;
                timekeeping_forward_now(tks);
                __timekeeping_inject_sleeptime(tks, delta);
                timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
        }

        /* Signal hrtimers about time change */
        clock_was_set(CLOCK_SET_WALL | CLOCK_SET_BOOT);
}
#endif

/**
 * timekeeping_resume - Resumes the generic timekeeping subsystem.
 */
void timekeeping_resume(void)
{
        struct timekeeper *tks = &tk_core.shadow_timekeeper;
        struct clocksource *clock = tks->tkr_mono.clock;
        struct timespec64 ts_new, ts_delta;
        bool inject_sleeptime = false;
        u64 cycle_now, nsec;
        unsigned long flags;

        read_persistent_clock64(&ts_new);

        clockevents_resume();
        clocksource_resume();

        raw_spin_lock_irqsave(&tk_core.lock, flags);

        /*
         * After system resumes, we need to calculate the suspended time and
         * compensate it for the OS time. There are 3 sources that could be
         * used: Nonstop clocksource during suspend, persistent clock and rtc
         * device.
         *
         * One specific platform may have 1 or 2 or all of them, and the
         * preference will be:
         *        suspend-nonstop clocksource -> persistent clock -> rtc
         * The less preferred source will only be tried if there is no better
         * usable source. The rtc part is handled separately in rtc core code.
         */
        cycle_now = tk_clock_read(&tks->tkr_mono);
        nsec = clocksource_stop_suspend_timing(clock, cycle_now);
        if (nsec > 0) {
                ts_delta = ns_to_timespec64(nsec);
                inject_sleeptime = true;
        } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
                ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
                inject_sleeptime = true;
        }

        if (inject_sleeptime) {
                suspend_timing_needed = false;
                __timekeeping_inject_sleeptime(tks, &ts_delta);
        }

        /* Re-base the last cycle value */
        tks->tkr_mono.cycle_last = cycle_now;
        tks->tkr_raw.cycle_last  = cycle_now;

        tks->ntp_error = 0;
        timekeeping_suspended = 0;
        timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
        raw_spin_unlock_irqrestore(&tk_core.lock, flags);

        touch_softlockup_watchdog();

        /* Resume the clockevent device(s) and hrtimers */
        tick_resume();
        /* Notify timerfd as resume is equivalent to clock_was_set() */
        timerfd_resume();
}

static void timekeeping_syscore_resume(void *data)
{
        timekeeping_resume();
}

int timekeeping_suspend(void)
{
        struct timekeeper *tks = &tk_core.shadow_timekeeper;
        struct timespec64 delta, delta_delta;
        static struct timespec64 old_delta;
        struct clocksource *curr_clock;
        unsigned long flags;
        u64 cycle_now;

        read_persistent_clock64(&timekeeping_suspend_time);

        /*
         * On some systems the persistent_clock can not be detected at
         * timekeeping_init by its return value, so if we see a valid
         * value returned, update the persistent_clock_exists flag.
         */
        if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
                persistent_clock_exists = true;

        suspend_timing_needed = true;

        raw_spin_lock_irqsave(&tk_core.lock, flags);
        timekeeping_forward_now(tks);
        timekeeping_suspended = 1;

        /*
         * Since we've called forward_now, cycle_last stores the value
         * just read from the current clocksource. Save this to potentially
         * use in suspend timing.
         */
        curr_clock = tks->tkr_mono.clock;
        cycle_now = tks->tkr_mono.cycle_last;
        clocksource_start_suspend_timing(curr_clock, cycle_now);

        if (persistent_clock_exists) {
                /*
                 * To avoid drift caused by repeated suspend/resumes,
                 * which each can add ~1 second drift error,
                 * try to compensate so the difference in system time
                 * and persistent_clock time stays close to constant.
                 */
                delta = timespec64_sub(tk_xtime(tks), timekeeping_suspend_time);
                delta_delta = timespec64_sub(delta, old_delta);
                if (abs(delta_delta.tv_sec) >= 2) {
                        /*
                         * if delta_delta is too large, assume time correction
                         * has occurred and set old_delta to the current delta.
                         */
                        old_delta = delta;
                } else {
                        /* Otherwise try to adjust old_system to compensate */
                        timekeeping_suspend_time =
                                timespec64_add(timekeeping_suspend_time, delta_delta);
                }
        }

        timekeeping_update_from_shadow(&tk_core, 0);
        halt_fast_timekeeper(tks);
        raw_spin_unlock_irqrestore(&tk_core.lock, flags);

        tick_suspend();
        clocksource_suspend();
        clockevents_suspend();

        return 0;
}

static int timekeeping_syscore_suspend(void *data)
{
        return timekeeping_suspend();
}

/* sysfs resume/suspend bits for timekeeping */
static const struct syscore_ops timekeeping_syscore_ops = {
        .resume                = timekeeping_syscore_resume,
        .suspend        = timekeeping_syscore_suspend,
};

static struct syscore timekeeping_syscore = {
        .ops = &timekeeping_syscore_ops,
};

static int __init timekeeping_init_ops(void)
{
        register_syscore(&timekeeping_syscore);
        return 0;
}
device_initcall(timekeeping_init_ops);

/*
 * Apply a multiplier adjustment to the timekeeper
 */
static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
                                                         s64 offset,
                                                         s32 mult_adj)
{
        s64 interval = tk->cycle_interval;

        if (mult_adj == 0) {
                return;
        } else if (mult_adj == -1) {
                interval = -interval;
                offset = -offset;
        } else if (mult_adj != 1) {
                interval *= mult_adj;
                offset *= mult_adj;
        }

        /*
         * So the following can be confusing.
         *
         * To keep things simple, lets assume mult_adj == 1 for now.
         *
         * When mult_adj != 1, remember that the interval and offset values
         * have been appropriately scaled so the math is the same.
         *
         * The basic idea here is that we're increasing the multiplier
         * by one, this causes the xtime_interval to be incremented by
         * one cycle_interval. This is because:
         *        xtime_interval = cycle_interval * mult
         * So if mult is being incremented by one:
         *        xtime_interval = cycle_interval * (mult + 1)
         * Its the same as:
         *        xtime_interval = (cycle_interval * mult) + cycle_interval
         * Which can be shortened to:
         *        xtime_interval += cycle_interval
         *
         * So offset stores the non-accumulated cycles. Thus the current
         * time (in shifted nanoseconds) is:
         *        now = (offset * adj) + xtime_nsec
         * Now, even though we're adjusting the clock frequency, we have
         * to keep time consistent. In other words, we can't jump back
         * in time, and we also want to avoid jumping forward in time.
         *
         * So given the same offset value, we need the time to be the same
         * both before and after the freq adjustment.
         *        now = (offset * adj_1) + xtime_nsec_1
         *        now = (offset * adj_2) + xtime_nsec_2
         * So:
         *        (offset * adj_1) + xtime_nsec_1 =
         *                (offset * adj_2) + xtime_nsec_2
         * And we know:
         *        adj_2 = adj_1 + 1
         * So:
         *        (offset * adj_1) + xtime_nsec_1 =
         *                (offset * (adj_1+1)) + xtime_nsec_2
         *        (offset * adj_1) + xtime_nsec_1 =
         *                (offset * adj_1) + offset + xtime_nsec_2
         * Canceling the sides:
         *        xtime_nsec_1 = offset + xtime_nsec_2
         * Which gives us:
         *        xtime_nsec_2 = xtime_nsec_1 - offset
         * Which simplifies to:
         *        xtime_nsec -= offset
         *
         * When subtracting offset from xtime_nsec, the same amount
         * (in appropriate units) has to be added to ntp_error, in
         * order to correctly track the delta between the time
         * reported in xtime_nsec, and the intended time.
         */
        if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
                /* NTP adjustment caused clocksource mult overflow */
                WARN_ON_ONCE(1);
                return;
        }

        tk->tkr_mono.mult += mult_adj;
        tk->xtime_interval += interval;
        tk->tkr_mono.xtime_nsec -= offset;
        tk->ntp_error += offset << tk->ntp_error_shift;
}

/*
 * Adjust the timekeeper's multiplier to the correct frequency
 * and also to reduce the accumulated error value.
 */
static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
{
        u64 ntp_tl = ntp_tick_length(tk->id);
        s64 skew = ntp_get_skew_delta(tk->id);
        u32 mult;

        /*
         * Determine the multiplier from the current NTP tick length plus
         * skew_delta. The skew biases mult so that ±1 dithering can deliver
         * the time_offset slew rate. Recompute when either changes.
         */
        if (likely(tk->ntp_tick == ntp_tl && tk->skew_delta == skew)) {
                /* Revert to the base mult rate. */
                mult = tk->tkr_mono.mult - tk->ntp_err_mult;
        } else {
                tk->ntp_tick = ntp_tl;
                tk->skew_delta = skew;
                /*
                 * skew_delta is stored pre-divided by HZ (matching time_offset);
                 * scale it back up to the full per-tick rate for the mult bias.
                 */
                skew *= NTP_INTERVAL_FREQ;
                mult = div64_u64((tk->ntp_tick + skew) >> tk->ntp_error_shift,
                                 tk->cycle_interval);
        }

        /*
         * If the clock is behind the NTP time, increase the multiplier by 1
         * to catch up with it. If it's ahead and there was a remainder in the
         * tick division, the clock will slow down. Otherwise it will stay
         * ahead until the tick length changes to a non-divisible value.
         */
        tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;
        mult += tk->ntp_err_mult;

        timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult);

        if (unlikely(tk->tkr_mono.clock->maxadj &&
                (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
                        > tk->tkr_mono.clock->maxadj))) {
                printk_once(KERN_WARNING
                        "Adjusting %s more than 11%% (%ld vs %ld)\n",
                        tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
                        (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
        }

        /*
         * It may be possible that when we entered this function, xtime_nsec
         * was very small.  Further, if we're slightly speeding the clocksource
         * in the code above, its possible the required corrective factor to
         * xtime_nsec could cause it to underflow.
         *
         * Now, since we have already accumulated the second and the NTP
         * subsystem has been notified via second_overflow(), we need to skip
         * the next update.
         */
        if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {
                tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<
                                                        tk->tkr_mono.shift;
                tk->xtime_sec--;
                tk->skip_second_overflow = 1;
        }
}

/*
 * accumulate_nsecs_to_secs - Accumulates nsecs into secs
 *
 * Helper function that accumulates the nsecs greater than a second
 * from the xtime_nsec field to the xtime_secs field.
 * It also calls into the NTP code to handle leapsecond processing.
 */
static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
{
        u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
        unsigned int clock_set = 0;

        while (tk->tkr_mono.xtime_nsec >= nsecps) {
                int leap;

                tk->tkr_mono.xtime_nsec -= nsecps;
                tk->xtime_sec++;

                /*
                 * Skip NTP update if this second was accumulated before,
                 * i.e. xtime_nsec underflowed in timekeeping_adjust()
                 */
                if (unlikely(tk->skip_second_overflow)) {
                        tk->skip_second_overflow = 0;
                        continue;
                }

                /* Figure out if its a leap sec and apply if needed */
                leap = second_overflow(tk->id, tk->xtime_sec);
                if (unlikely(leap)) {
                        struct timespec64 ts;

                        tk->xtime_sec += leap;

                        ts.tv_sec = leap;
                        ts.tv_nsec = 0;
                        tk_set_wall_to_mono(tk,
                                timespec64_sub(tk->wall_to_monotonic, ts));

                        __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);

                        clock_set = TK_CLOCK_WAS_SET;
                }
        }
        return clock_set;
}

/*
 * logarithmic_accumulation - shifted accumulation of cycles
 *
 * This functions accumulates a shifted interval of cycles into
 * a shifted interval nanoseconds. Allows for O(log) accumulation
 * loop.
 *
 * Returns the unconsumed cycles.
 */
static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
                                    u32 shift, unsigned int *clock_set)
{
        u64 interval = tk->cycle_interval << shift;
        u64 snsec_per_sec;

        /* If the offset is smaller than a shifted interval, do nothing */
        if (offset < interval)
                return offset;

        /* Accumulate one shifted interval */
        offset -= interval;
        tk->tkr_mono.cycle_last += interval;
        tk->tkr_raw.cycle_last  += interval;

        tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
        *clock_set |= accumulate_nsecs_to_secs(tk);

        /* Accumulate raw time */
        tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;
        snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
        while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {
                tk->tkr_raw.xtime_nsec -= snsec_per_sec;
                tk->raw_sec++;
        }

        /* Accumulate error between NTP and clock interval */
        tk->ntp_error += tk->ntp_tick << shift;
        tk->ntp_error -= tk->xtime_interval << (tk->ntp_error_shift + shift);

        /*
         * When skewing, do so by adjusting ntp_error to impart an extra
         * target delta into ntp_error per tick, limited to what can be
         * drained from time_offset / time_adjust to avoid overshoot.
         *
         * The base 'mult' value was calculated with the skew taken into
         * account, such that the per-tick choice of 'mult' vs. 'mult+1'
         * allows for the desired effective rate and ntp_error does not
         * grow unbounded.
         *
         * Once the full desired phase offset is delivered, any remaining
         * skew imparted by the adjusted 'mult', accounted above, remains
         * in ntp_error and will be compensated by the dithering over time.
         */
        if (tk->skew_delta)
                tk->ntp_error += ntp_drain_skew(tk->id, tk->skew_delta << shift,
                                                shift) * NTP_INTERVAL_FREQ;

        return offset;
}

/*
 * timekeeping_advance - Updates the timekeeper to the current time and
 * current NTP tick length
 */
static bool __timekeeping_advance(struct tk_data *tkd, enum timekeeping_adv_mode mode)
{
        struct timekeeper *tk = &tkd->shadow_timekeeper;
        struct timekeeper *real_tk = &tkd->timekeeper;
        unsigned int clock_set = 0;
        int shift = 0, maxshift;
        u64 offset, orig_offset;

        /* Make sure we're fully resumed: */
        if (unlikely(timekeeping_suspended))
                return false;

        offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
                                   tk->tkr_mono.cycle_last, tk->tkr_mono.mask,
                                   tk->tkr_mono.clock->max_raw_delta);
        orig_offset = offset;
        /* Check if there's really nothing to do */
        if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)
                return false;

        /*
         * With NO_HZ we may have to accumulate many cycle_intervals
         * (think "ticks") worth of time at once. To do this efficiently,
         * we calculate the largest doubling multiple of cycle_intervals
         * that is smaller than the offset.  We then accumulate that
         * chunk in one go, and then try to consume the next smaller
         * doubled multiple.
         */
        shift = ilog2(offset) - ilog2(tk->cycle_interval);
        shift = max(0, shift);
        /* Bound shift to one less than what overflows tick_length */
        maxshift = (64 - (ilog2(ntp_tick_length(tk->id)) + 1)) - 1;
        shift = min(shift, maxshift);
        while (offset >= tk->cycle_interval) {
                offset = logarithmic_accumulation(tk, offset, shift, &clock_set);
                if (offset < tk->cycle_interval<<shift)
                        shift--;
        }

        /* Adjust the multiplier to correct NTP error */
        timekeeping_adjust(tk, offset);

        /*
         * Finally, make sure that after the rounding
         * xtime_nsec isn't larger than NSEC_PER_SEC
         */
        clock_set |= accumulate_nsecs_to_secs(tk);

        /*
         * To avoid inconsistencies caused adjtimex TK_ADV_FREQ calls
         * making small negative adjustments to the base xtime_nsec
         * value, only update the coarse clocks if we accumulated time
         */
        if (orig_offset != offset)
                tk_update_coarse_nsecs(tk);

        timekeeping_update_from_shadow(tkd, clock_set);

        return !!clock_set;
}

static bool timekeeping_advance(enum timekeeping_adv_mode mode)
{
        guard(raw_spinlock_irqsave)(&tk_core.lock);
        return __timekeeping_advance(&tk_core, mode);
}

/**
 * update_wall_time - Uses the current clocksource to increment the wall time
 *
 * It also updates the enabled auxiliary clock timekeepers
 */
void update_wall_time(void)
{
        if (timekeeping_advance(TK_ADV_TICK))
                clock_was_set_delayed();
        tk_aux_advance();
}

/**
 * getboottime64 - Return the real time of system boot.
 * @ts:                pointer to the timespec64 to be set
 *
 * Returns the wall-time of boot in a timespec64.
 *
 * This is based on the wall_to_monotonic offset and the total suspend
 * time. Calls to settimeofday will affect the value returned (which
 * basically means that however wrong your real time clock is at boot time,
 * you get the right time here).
 */
void getboottime64(struct timespec64 *ts)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);

        *ts = ktime_to_timespec64(t);
}
EXPORT_SYMBOL_GPL(getboottime64);

void ktime_get_coarse_real_ts64(struct timespec64 *ts)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        unsigned int seq;

        do {
                seq = read_seqcount_begin(&tk_core.seq);

                *ts = tk_xtime_coarse(tk);
        } while (read_seqcount_retry(&tk_core.seq, seq));
}
EXPORT_SYMBOL(ktime_get_coarse_real_ts64);

/**
 * ktime_get_coarse_real_ts64_mg - return latter of coarse grained time or floor
 * @ts:                timespec64 to be filled
 *
 * Fetch the global mg_floor value, convert it to realtime and compare it
 * to the current coarse-grained time. Fill @ts with whichever is
 * latest. Note that this is a filesystem-specific interface and should be
 * avoided outside of that context.
 */
void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        u64 floor = atomic64_read(&mg_floor);
        ktime_t f_real, offset, coarse;
        unsigned int seq;

        do {
                seq = read_seqcount_begin(&tk_core.seq);
                *ts = tk_xtime_coarse(tk);
                offset = tk_core.timekeeper.offs_real;
        } while (read_seqcount_retry(&tk_core.seq, seq));

        coarse = timespec64_to_ktime(*ts);
        f_real = ktime_add(floor, offset);
        if (ktime_after(f_real, coarse))
                *ts = ktime_to_timespec64(f_real);
}

/**
 * ktime_get_real_ts64_mg - attempt to update floor value and return result
 * @ts:                pointer to the timespec to be set
 *
 * Get a monotonic fine-grained time value and attempt to swap it into
 * mg_floor. If that succeeds then accept the new floor value. If it fails
 * then another task raced in during the interim time and updated the
 * floor.  Since any update to the floor must be later than the previous
 * floor, either outcome is acceptable.
 *
 * Typically this will be called after calling ktime_get_coarse_real_ts64_mg(),
 * and determining that the resulting coarse-grained timestamp did not effect
 * a change in ctime. Any more recent floor value would effect a change to
 * ctime, so there is no need to retry the atomic64_try_cmpxchg() on failure.
 *
 * @ts will be filled with the latest floor value, regardless of the outcome of
 * the cmpxchg. Note that this is a filesystem specific interface and should be
 * avoided outside of that context.
 */
void ktime_get_real_ts64_mg(struct timespec64 *ts)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        ktime_t old = atomic64_read(&mg_floor);
        ktime_t offset, mono;
        unsigned int seq;
        u64 nsecs;

        do {
                seq = read_seqcount_begin(&tk_core.seq);

                ts->tv_sec = tk->xtime_sec;
                mono = tk->tkr_mono.base;
                nsecs = timekeeping_get_ns(&tk->tkr_mono);
                offset = tk_core.timekeeper.offs_real;
        } while (read_seqcount_retry(&tk_core.seq, seq));

        mono = ktime_add_ns(mono, nsecs);

        /*
         * Attempt to update the floor with the new time value. As any
         * update must be later then the existing floor, and would effect
         * a change to ctime from the perspective of the current task,
         * accept the resulting floor value regardless of the outcome of
         * the swap.
         */
        if (atomic64_try_cmpxchg(&mg_floor, &old, mono)) {
                ts->tv_nsec = 0;
                timespec64_add_ns(ts, nsecs);
                timekeeping_inc_mg_floor_swaps();
        } else {
                /*
                 * Another task changed mg_floor since "old" was fetched.
                 * "old" has been updated with the latest value of "mg_floor".
                 * That value is newer than the previous floor value, which
                 * is enough to effect a change to ctime. Accept it.
                 */
                *ts = ktime_to_timespec64(ktime_add(old, offset));
        }
}

void ktime_get_coarse_ts64(struct timespec64 *ts)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        struct timespec64 now, mono;
        unsigned int seq;

        do {
                seq = read_seqcount_begin(&tk_core.seq);

                now = tk_xtime_coarse(tk);
                mono = tk->wall_to_monotonic;
        } while (read_seqcount_retry(&tk_core.seq, seq));

        set_normalized_timespec64(ts, now.tv_sec + mono.tv_sec,
                                  now.tv_nsec + mono.tv_nsec);
}
EXPORT_SYMBOL(ktime_get_coarse_ts64);

/*
 * Must hold jiffies_lock
 */
void do_timer(unsigned long ticks)
{
        jiffies_64 += ticks;
        calc_global_load();
}

/**
 * ktime_get_update_offsets_now - hrtimer helper
 * @cwsseq:        pointer to check and store the clock was set sequence number
 * @offs_real:        pointer to storage for monotonic -> realtime offset
 * @offs_boot:        pointer to storage for monotonic -> boottime offset
 * @offs_tai:        pointer to storage for monotonic -> clock tai offset
 *
 * Returns current monotonic time and updates the offsets if the
 * sequence number in @cwsseq and timekeeper.clock_was_set_seq are
 * different.
 *
 * Called from hrtimer_interrupt() or retrigger_next_event()
 */
ktime_t ktime_get_update_offsets_now(u32 *cwsseq, ktime_t *offs_real,
                                     ktime_t *offs_boot, ktime_t *offs_tai)
{
        struct timekeeper *tk = &tk_core.timekeeper;
        unsigned int seq;
        ktime_t base;
        u64 nsecs;

        do {
                seq = read_seqcount_begin(&tk_core.seq);

                base = tk->tkr_mono.base;
                nsecs = timekeeping_get_ns(&tk->tkr_mono);
                base = ktime_add_ns(base, nsecs);

                if (*cwsseq != tk->clock_was_set_seq) {
                        *cwsseq = tk->clock_was_set_seq;
                        *offs_real = tk->offs_real;
                        *offs_boot = tk->offs_boot;
                        *offs_tai = tk->offs_tai;
                }

                /* Handle leapsecond insertion adjustments */
                if (unlikely(base >= tk->next_leap_ktime))
                        *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));

        } while (read_seqcount_retry(&tk_core.seq, seq));

        return base;
}

/*
 * timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
 */
static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux_clock)
{
        if (txc->modes & ADJ_ADJTIME) {
                /* singleshot must not be used with any other mode bits */
                if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))
                        return -EINVAL;
                if (!(txc->modes & ADJ_OFFSET_READONLY) &&
                    !capable(CAP_SYS_TIME))
                        return -EPERM;
        } else {
                /* In order to modify anything, you gotta be super-user! */
                if (txc->modes && !capable(CAP_SYS_TIME))
                        return -EPERM;
                /*
                 * if the quartz is off by more than 10% then
                 * something is VERY wrong!
                 */
                if (txc->modes & ADJ_TICK &&
                    (txc->tick <  900000/USER_HZ ||
                     txc->tick > 1100000/USER_HZ))
                        return -EINVAL;
        }

        if (txc->modes & ADJ_SETOFFSET) {
                /* In order to inject time, you gotta be super-user! */
                if (!capable(CAP_SYS_TIME))
                        return -EPERM;

                /*
                 * Validate if a timespec/timeval used to inject a time
                 * offset is valid.  Offsets can be positive or negative, so
                 * we don't check tv_sec. The value of the timeval/timespec
                 * is the sum of its fields,but *NOTE*:
                 * The field tv_usec/tv_nsec must always be non-negative and
                 * we can't have more nanoseconds/microseconds than a second.
                 */
                if (txc->time.tv_usec < 0)
                        return -EINVAL;

                if (txc->modes & ADJ_NANO) {
                        if (txc->time.tv_usec >= NSEC_PER_SEC)
                                return -EINVAL;
                } else {
                        if (txc->time.tv_usec >= USEC_PER_SEC)
                                return -EINVAL;
                }
        }

        /*
         * Check for potential multiplication overflows that can
         * only happen on 64-bit systems:
         */
        if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) {
                if (LLONG_MIN / PPM_SCALE > txc->freq)
                        return -EINVAL;
                if (LLONG_MAX / PPM_SCALE < txc->freq)
                        return -EINVAL;
        }

        if (aux_clock) {
                /* Auxiliary clocks are similar to TAI and do not have leap seconds */
                if (txc->modes & ADJ_STATUS &&
                    txc->status & (STA_INS | STA_DEL))
                        return -EINVAL;

                /* No TAI offset setting */
                if (txc->modes & ADJ_TAI)
                        return -EINVAL;

                /* No PPS support either */
                if (txc->modes & ADJ_STATUS &&
                    txc->status & (STA_PPSFREQ | STA_PPSTIME))
                        return -EINVAL;
        }

        return 0;
}

/**
 * random_get_entropy_fallback - Returns the raw clock source value,
 * used by random.c for platforms with no valid random_get_entropy().
 */
unsigned long random_get_entropy_fallback(void)
{
        struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;
        struct clocksource *clock = READ_ONCE(tkr->clock);

        if (unlikely(timekeeping_suspended || !clock))
                return 0;
        return clock->read(clock);
}
EXPORT_SYMBOL_GPL(random_get_entropy_fallback);

struct adjtimex_result {
        struct audit_ntp_data        ad;
        struct timespec64        delta;
        bool                        clock_set;
};

static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc,
                         struct adjtimex_result *result)
{
        struct timekeeper *tks = &tkd->shadow_timekeeper;
        bool aux_clock = !timekeeper_is_core_tk(tks);
        struct timespec64 ts;
        s32 orig_tai, tai;
        int ret;

        /* Validate the data before disabling interrupts */
        ret = timekeeping_validate_timex(txc, aux_clock);
        if (ret)
                return ret;
        add_device_randomness(txc, sizeof(*txc));

        if (!aux_clock) {
                ktime_get_real_ts64(&ts);
        } else {
                if (!tk_get_aux_ts64(tkd->timekeeper.id, &ts))
                        return -ENODEV;
        }

        add_device_randomness(&ts, sizeof(ts));

        guard(raw_spinlock_irqsave)(&tkd->lock);

        if (!tks->clock_valid)
                return -ENODEV;

        if (txc->modes & ADJ_SETOFFSET) {
                result->delta.tv_sec  = txc->time.tv_sec;
                result->delta.tv_nsec = txc->time.tv_usec;
                if (!(txc->modes & ADJ_NANO))
                        result->delta.tv_nsec *= 1000;
                ret = __timekeeping_inject_offset(tkd, &result->delta);
                if (ret)
                        return ret;
                result->clock_set = true;
        }

        orig_tai = tai = tks->tai_offset;
        ret = ntp_adjtimex(tks->id, txc, &ts, &tai, &result->ad);

        if (tai != orig_tai) {
                __timekeeping_set_tai_offset(tks, tai);
                timekeeping_update_from_shadow(tkd, TK_CLOCK_WAS_SET);
                result->clock_set = true;
        } else {
                tk_update_leap_state_all(tkd);
        }

        /* Update the multiplier immediately if frequency was set directly */
        if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))
                result->clock_set |= __timekeeping_advance(tkd, TK_ADV_FREQ);

        return ret;
}

/**
 * do_adjtimex() - Accessor function to NTP __do_adjtimex function
 * @txc:        Pointer to kernel_timex structure containing NTP parameters
 */
int do_adjtimex(struct __kernel_timex *txc)
{
        struct adjtimex_result result = { };
        int ret;

        ret = __do_adjtimex(&tk_core, txc, &result);
        if (ret < 0)
                return ret;

        if (txc->modes & ADJ_SETOFFSET)
                audit_tk_injoffset(result.delta);

        audit_ntp_log(&result.ad);

        if (result.clock_set)
                clock_was_set(CLOCK_SET_WALL);

        ntp_notify_cmos_timer(result.delta.tv_sec != 0);

        return ret;
}

/*
 * Invoked from NTP with the time keeper lock held, so lockless access is
 * fine.
 */
long ktime_get_ntp_seconds(unsigned int id)
{
        return timekeeper_data[id].timekeeper.xtime_sec;
}

#ifdef CONFIG_NTP_PPS
/**
 * hardpps() - Accessor function to NTP __hardpps function
 * @phase_ts:        Pointer to timespec64 structure representing phase timestamp
 * @raw_ts:        Pointer to timespec64 structure representing raw timestamp
 */
void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
{
        guard(raw_spinlock_irqsave)(&tk_core.lock);
        __hardpps(phase_ts, raw_ts);
}
EXPORT_SYMBOL(hardpps);
#endif /* CONFIG_NTP_PPS */

#ifdef CONFIG_POSIX_AUX_CLOCKS
#include "posix-timers.h"

/*
 * Bitmap for the activated auxiliary timekeepers to allow lockless quick
 * checks in the hot paths without touching extra cache lines. If set, then
 * the state of the corresponding timekeeper has to be re-checked under
 * timekeeper::lock.
 */
static unsigned long aux_timekeepers;

static inline unsigned int clockid_to_tkid(unsigned int id)
{
        return TIMEKEEPER_AUX_FIRST + id - CLOCK_AUX;
}

static inline struct tk_data *aux_get_tk_data(clockid_t id)
{
        if (!clockid_is_aux_clock(id))
                return NULL;
        return &timekeeper_data[clockid_to_tkid(id)];
}

/* Invoked from timekeeping after a clocksource change */
static void tk_aux_update_clocksource(void)
{
        unsigned long active = READ_ONCE(aux_timekeepers);
        unsigned int id;

        for_each_set_bit(id, &active, BITS_PER_LONG) {
                struct tk_data *tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST];
                struct timekeeper *tks = &tkd->shadow_timekeeper;

                guard(raw_spinlock_irqsave)(&tkd->lock);
                if (!tks->clock_valid)
                        continue;

                timekeeping_forward_now(tks);
                tk_setup_internals(tks, tk_core.timekeeper.tkr_raw.clock);
                timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL);
        }
}

static void tk_aux_advance(void)
{
        unsigned long active = READ_ONCE(aux_timekeepers);
        unsigned int id;

        /* Lockless quick check to avoid extra cache lines */
        for_each_set_bit(id, &active, BITS_PER_LONG) {
                struct tk_data *aux_tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST];

                guard(raw_spinlock)(&aux_tkd->lock);
                if (aux_tkd->shadow_timekeeper.clock_valid)
                        __timekeeping_advance(aux_tkd, TK_ADV_TICK);
        }
}

/**
 * ktime_get_aux - Get time for a AUX clock
 * @id:        ID of the clock to read (CLOCK_AUX...)
 * @kt:        Pointer to ktime_t to store the time stamp
 *
 * Returns: True if the timestamp is valid, false otherwise
 */
bool ktime_get_aux(clockid_t id, ktime_t *kt)
{
        struct tk_data *aux_tkd = aux_get_tk_data(id);
        struct timekeeper *aux_tk;
        unsigned int seq;
        ktime_t base;
        u64 nsecs;

        WARN_ON(timekeeping_suspended);

        if (!aux_tkd)
                return false;

        aux_tk = &aux_tkd->timekeeper;
        do {
                seq = read_seqcount_begin(&aux_tkd->seq);
                if (!aux_tk->clock_valid)
                        return false;

                base = ktime_add(aux_tk->tkr_mono.base, aux_tk->offs_aux);
                nsecs = timekeeping_get_ns(&aux_tk->tkr_mono);
        } while (read_seqcount_retry(&aux_tkd->seq, seq));

        *kt = ktime_add_ns(base, nsecs);
        return true;
}
EXPORT_SYMBOL_GPL(ktime_get_aux);

/**
 * ktime_get_aux_ts64 - Get time for a AUX clock
 * @id:        ID of the clock to read (CLOCK_AUX...)
 * @ts:        Pointer to timespec64 to store the time stamp
 *
 * Returns: True if the timestamp is valid, false otherwise
 */
bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *ts)
{
        ktime_t now;

        if (!ktime_get_aux(id, &now))
                return false;
        *ts = ktime_to_timespec64(now);
        return true;
}
EXPORT_SYMBOL_GPL(ktime_get_aux_ts64);

static int aux_get_res(clockid_t id, struct timespec64 *tp)
{
        if (!clockid_is_aux_clock(id))
                return -ENODEV;

        tp->tv_sec = aux_clock_resolution_ns() / NSEC_PER_SEC;
        tp->tv_nsec = aux_clock_resolution_ns() % NSEC_PER_SEC;
        return 0;
}

static int aux_get_timespec(clockid_t id, struct timespec64 *tp)
{
        return ktime_get_aux_ts64(id, tp) ? 0 : -ENODEV;
}

static int aux_clock_set(const clockid_t id, const struct timespec64 *tnew)
{
        struct tk_data *aux_tkd = aux_get_tk_data(id);
        struct timekeeper *aux_tks;
        ktime_t tnow, nsecs;

        if (!timespec64_valid_settod(tnew))
                return -EINVAL;
        if (!aux_tkd)
                return -ENODEV;

        aux_tks = &aux_tkd->shadow_timekeeper;

        guard(raw_spinlock_irq)(&aux_tkd->lock);
        if (!aux_tks->clock_valid)
                return -ENODEV;

        /* Forward the timekeeper base time */
        timekeeping_forward_now(aux_tks);
        /*
         * Get the updated base time. tkr_mono.base has not been
         * updated yet, so do that first. That makes the update
         * in timekeeping_update_from_shadow() redundant, but
         * that's harmless. After that @tnow can be calculated
         * by using tkr_mono::cycle_last, which has been set
         * by timekeeping_forward_now().
         */
        tk_update_ktime_data(aux_tks);
        nsecs = timekeeping_cycles_to_ns(&aux_tks->tkr_mono, aux_tks->tkr_mono.cycle_last);
        tnow = ktime_add(aux_tks->tkr_mono.base, nsecs);

        /*
         * Calculate the new AUX offset as delta to @tnow ("monotonic").
         * That avoids all the tk::xtime back and forth conversions as
         * xtime ("realtime") is not applicable for auxiliary clocks and
         * kept in sync with "monotonic".
         */
        tk_update_aux_offs(aux_tks, ktime_sub(timespec64_to_ktime(*tnew), tnow));

        timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
        return 0;
}

static int aux_clock_adj(const clockid_t id, struct __kernel_timex *txc)
{
        struct tk_data *aux_tkd = aux_get_tk_data(id);
        struct adjtimex_result result = { };

        if (!aux_tkd)
                return -ENODEV;

        /*
         * @result is ignored for now as there are neither hrtimers nor a
         * RTC related to auxiliary clocks for now.
         */
        return __do_adjtimex(aux_tkd, txc, &result);
}

const struct k_clock clock_aux = {
        .clock_getres                = aux_get_res,
        .clock_get_timespec        = aux_get_timespec,
        .clock_set                = aux_clock_set,
        .clock_adj                = aux_clock_adj,
};

static void aux_clock_enable(clockid_t id)
{
        struct tk_read_base *tkr_raw = &tk_core.timekeeper.tkr_raw;
        struct tk_data *aux_tkd = aux_get_tk_data(id);
        struct timekeeper *aux_tks = &aux_tkd->shadow_timekeeper;

        /* Prevent the core timekeeper from changing. */
        guard(raw_spinlock_irq)(&tk_core.lock);

        /*
         * Setup the auxiliary clock assuming that the raw core timekeeper
         * clock frequency conversion is close enough. Userspace has to
         * adjust for the deviation via clock_adjtime(2).
         */
        guard(raw_spinlock_nested)(&aux_tkd->lock);

        /* Remove leftovers of a previous registration */
        memset(aux_tks, 0, sizeof(*aux_tks));
        /* Restore the timekeeper id */
        aux_tks->id = aux_tkd->timekeeper.id;
        /* Setup the timekeeper based on the current system clocksource */
        tk_setup_internals(aux_tks, tkr_raw->clock);

        /* Mark it valid and set it live */
        aux_tks->clock_valid = true;
        timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
}

static void aux_clock_disable(clockid_t id)
{
        struct tk_data *aux_tkd = aux_get_tk_data(id);

        guard(raw_spinlock_irq)(&aux_tkd->lock);
        aux_tkd->shadow_timekeeper.clock_valid = false;
        timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
}

static DEFINE_MUTEX(aux_clock_mutex);

static ssize_t aux_clock_enable_store(struct kobject *kobj, struct kobj_attribute *attr,
                                      const char *buf, size_t count)
{
        /* Lazy atoi() as name is "0..7" */
        int id = kobj->name[0] & 0x7;
        bool enable;

        if (!capable(CAP_SYS_TIME))
                return -EPERM;

        if (kstrtobool(buf, &enable) < 0)
                return -EINVAL;

        guard(mutex)(&aux_clock_mutex);
        if (enable == test_bit(id, &aux_timekeepers))
                return count;

        if (enable) {
                aux_clock_enable(CLOCK_AUX + id);
                set_bit(id, &aux_timekeepers);
        } else {
                aux_clock_disable(CLOCK_AUX + id);
                clear_bit(id, &aux_timekeepers);
        }
        return count;
}

static ssize_t aux_clock_enable_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
{
        unsigned long active = READ_ONCE(aux_timekeepers);
        /* Lazy atoi() as name is "0..7" */
        int id = kobj->name[0] & 0x7;

        return sysfs_emit(buf, "%d\n", test_bit(id, &active));
}

static struct kobj_attribute aux_clock_enable_attr = __ATTR_RW(aux_clock_enable);

static struct attribute *aux_clock_enable_attrs[] = {
        &aux_clock_enable_attr.attr,
        NULL
};

static const struct attribute_group aux_clock_enable_attr_group = {
        .attrs = aux_clock_enable_attrs,
};

static int __init tk_aux_sysfs_init(void)
{
        struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj);
        struct kobject *clks[MAX_AUX_CLOCKS];
        int ret = -ENOMEM;
        int i;

        if (!tko)
                return ret;

        auxo = kobject_create_and_add("aux_clocks", tko);
        if (!auxo)
                goto err_clean;

        for (i = 0; i < MAX_AUX_CLOCKS; i++) {
                char id[2] = { [0] = '0' + i, };
                clks[i] = kobject_create_and_add(id, auxo);

                if (!clks[i]) {
                        ret = -ENOMEM;
                        goto err_clks;
                }

                ret = sysfs_create_group(clks[i], &aux_clock_enable_attr_group);
                if (ret)
                        goto err_clk;
        }
        return 0;

err_clk:
        kobject_put(clks[i]);
err_clks:
        while (--i >= 0) {
                sysfs_remove_group(clks[i], &aux_clock_enable_attr_group);
                kobject_put(clks[i]);
        }
err_clean:
        kobject_put(auxo);
        kobject_put(tko);
        return ret;
}
late_initcall(tk_aux_sysfs_init);

static __init void tk_aux_setup(void)
{
        for (int i = TIMEKEEPER_AUX_FIRST; i <= TIMEKEEPER_AUX_LAST; i++)
                tkd_basic_setup(&timekeeper_data[i], i, false);
}
#endif /* CONFIG_POSIX_AUX_CLOCKS */














































  771 

 4525 
   97 










  478 
 3918 

















   18 
   35 

   18 















  178 

  475 





   74 
  302 













   25 




  983 
    9 












   10 
    1 








   80 






   85 




    9 

    9 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (c) 2020 Christoph Hellwig.
 *
 * Support for "universal" pointers that can point to either kernel or userspace
 * memory.
 */
#ifndef _LINUX_SOCKPTR_H
#define _LINUX_SOCKPTR_H

#include <linux/slab.h>
#include <linux/uaccess.h>

typedef struct {
        union {
                void                *kernel;
                void __user        *user;
        };
        bool                is_kernel : 1;
} sockptr_t;

static inline bool sockptr_is_kernel(sockptr_t sockptr)
{
        return sockptr.is_kernel;
}

static inline sockptr_t KERNEL_SOCKPTR(void *p)
{
        return (sockptr_t) { .kernel = p, .is_kernel = true };
}

static inline sockptr_t USER_SOCKPTR(void __user *p)
{
        return (sockptr_t) { .user = p };
}

static inline bool sockptr_is_null(sockptr_t sockptr)
{
        if (sockptr_is_kernel(sockptr))
                return !sockptr.kernel;
        return !sockptr.user;
}

static inline int copy_from_sockptr_offset(void *dst, sockptr_t src,
                size_t offset, size_t size)
{
        if (!sockptr_is_kernel(src))
                return copy_from_user(dst, src.user + offset, size);
        memcpy(dst, src.kernel + offset, size);
        return 0;
}

/* Deprecated.
 * This is unsafe, unless caller checked user provided optlen.
 * Prefer copy_safe_from_sockptr() instead.
 *
 * Returns 0 for success, or number of bytes not copied on error.
 */
static inline int copy_from_sockptr(void *dst, sockptr_t src, size_t size)
{
        return copy_from_sockptr_offset(dst, src, 0, size);
}

/**
 * copy_safe_from_sockptr: copy a struct from sockptr
 * @dst:   Destination address, in kernel space. This buffer must be @ksize
 *         bytes long.
 * @ksize: Size of @dst struct.
 * @optval: Source address. (in user or kernel space)
 * @optlen: Size of @optval data.
 *
 * Returns:
 *  * -EINVAL: @optlen < @ksize
 *  * -EFAULT: access to userspace failed.
 *  * 0 : @ksize bytes were copied
 */
static inline int copy_safe_from_sockptr(void *dst, size_t ksize,
                                         sockptr_t optval, unsigned int optlen)
{
        if (optlen < ksize)
                return -EINVAL;
        if (copy_from_sockptr(dst, optval, ksize))
                return -EFAULT;
        return 0;
}

static inline int copy_struct_from_sockptr(void *dst, size_t ksize,
                sockptr_t src, size_t usize)
{
        if (!sockptr_is_kernel(src))
                return copy_struct_from_user(dst, ksize, src.user, usize);

        return copy_struct_from_bounce_buffer(dst, ksize, src.kernel, usize);
}

static inline int copy_to_sockptr_offset(sockptr_t dst, size_t offset,
                const void *src, size_t size)
{
        if (!sockptr_is_kernel(dst))
                return copy_to_user(dst.user + offset, src, size);
        memcpy(dst.kernel + offset, src, size);
        return 0;
}

static inline int copy_to_sockptr(sockptr_t dst, const void *src, size_t size)
{
        return copy_to_sockptr_offset(dst, 0, src, size);
}

static inline int
copy_struct_to_sockptr(sockptr_t dst, size_t usize, const void *src,
                       size_t ksize, bool *ignored_trailing)
{
        if (!sockptr_is_kernel(dst))
                return copy_struct_to_user(dst.user, usize, src, ksize, ignored_trailing);

        return copy_struct_to_bounce_buffer(dst.kernel, usize, src, ksize, ignored_trailing);
}

static inline void *memdup_sockptr_noprof(sockptr_t src, size_t len)
{
        void *p = kmalloc_track_caller_noprof(len, GFP_USER | __GFP_NOWARN);

        if (!p)
                return ERR_PTR(-ENOMEM);
        if (copy_from_sockptr(p, src, len)) {
                kfree(p);
                return ERR_PTR(-EFAULT);
        }
        return p;
}
#define memdup_sockptr(...)        alloc_hooks(memdup_sockptr_noprof(__VA_ARGS__))

static inline void *memdup_sockptr_nul_noprof(sockptr_t src, size_t len)
{
        char *p = kmalloc_track_caller_noprof(len + 1, GFP_KERNEL);

        if (!p)
                return ERR_PTR(-ENOMEM);
        if (copy_from_sockptr(p, src, len)) {
                kfree(p);
                return ERR_PTR(-EFAULT);
        }
        p[len] = '\0';
        return p;
}
#define memdup_sockptr_nul(...)        alloc_hooks(memdup_sockptr_nul_noprof(__VA_ARGS__))

static inline long strncpy_from_sockptr(char *dst, sockptr_t src, size_t count)
{
        if (sockptr_is_kernel(src)) {
                size_t len = min(strnlen(src.kernel, count - 1) + 1, count);

                memcpy(dst, src.kernel, len);
                return len;
        }
        return strncpy_from_user(dst, src.user, count);
}

static inline int check_zeroed_sockptr(sockptr_t src, size_t offset,
                                       size_t size)
{
        if (!sockptr_is_kernel(src))
                return check_zeroed_user(src.user + offset, size);
        return memchr_inv(src.kernel + offset, 0, size) == NULL;
}

#endif /* _LINUX_SOCKPTR_H */































   52 
   52 

















   51 

   51 
   51 
   51 




   51 





   51 


   51 




   50 
   50 



   28 
   22 
    1 









   50 

    7 






    8 




   21 
    7 




    3 

   12 


   51 




  133 




























  133 




   33 










   86 



   86 



    8 
   81 





  106 



   96 
   73 
   46 



   55 





    8 
   52 

   55 





   35 








   35 







   13 



    2 











   11 







    1 








   86 







   86 



   55 


   12 


    1 


   33 

   42 







  220 







































   32 
   32 




























   36 


   36 












   27 






   20 





































































































  135 























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Device handling code
 *        Linux ethernet bridge
 *
 *        Authors:
 *        Lennert Buytenhek                <buytenh@gnu.org>
 */

#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/netpoll.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/list.h>
#include <linux/netfilter_bridge.h>

#include <linux/uaccess.h>
#include <net/netdev_lock.h>

#include "br_private.h"

#define COMMON_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA | \
                         NETIF_F_GSO_MASK | NETIF_F_HW_CSUM)

const struct nf_br_ops __rcu *nf_br_ops __read_mostly;
EXPORT_SYMBOL_GPL(nf_br_ops);

/* net device transmit always called with BH disabled */
netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev)
{
        enum skb_drop_reason reason = pskb_may_pull_reason(skb, ETH_HLEN);
        struct net_bridge_mcast_port *pmctx_null = NULL;
        struct net_bridge *br = netdev_priv(dev);
        struct net_bridge_mcast *brmctx = &br->multicast_ctx;
        struct net_bridge_fdb_entry *dst;
        struct net_bridge_mdb_entry *mdst;
        const struct nf_br_ops *nf_ops;
        u8 state = BR_STATE_FORWARDING;
        struct net_bridge_vlan *vlan;
        const unsigned char *dest;
        u16 vid = 0;

        if (unlikely(reason != SKB_NOT_DROPPED_YET)) {
                kfree_skb_reason(skb, reason);
                return NETDEV_TX_OK;
        }

        memset(skb->cb, 0, sizeof(struct br_input_skb_cb));
        br_tc_skb_miss_set(skb, false);

        rcu_read_lock();
        nf_ops = rcu_dereference(nf_br_ops);
        if (nf_ops && nf_ops->br_dev_xmit_hook(skb)) {
                rcu_read_unlock();
                return NETDEV_TX_OK;
        }

        dev_sw_netstats_tx_add(dev, 1, skb->len);

        br_switchdev_frame_unmark(skb);
        BR_INPUT_SKB_CB(skb)->brdev = dev;
        BR_INPUT_SKB_CB(skb)->frag_max_size = 0;

        skb_reset_mac_header(skb);
        skb_pull(skb, ETH_HLEN);

        if (!br_allowed_ingress(br, br_vlan_group_rcu(br), skb, &vid,
                                &state, &vlan))
                goto out;

        if (IS_ENABLED(CONFIG_INET) &&
            (eth_hdr(skb)->h_proto == htons(ETH_P_ARP) ||
             eth_hdr(skb)->h_proto == htons(ETH_P_RARP)) &&
            br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED)) {
                br_do_proxy_suppress_arp(skb, br, vid, NULL);
        } else if (ipv6_mod_enabled() &&
                   skb->protocol == htons(ETH_P_IPV6) &&
                   br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED) &&
                   pskb_may_pull(skb, sizeof(struct ipv6hdr) +
                                 sizeof(struct nd_msg)) &&
                   ipv6_hdr(skb)->nexthdr == IPPROTO_ICMPV6) {
                        struct nd_msg *msg;

                        msg = br_is_nd_neigh_msg(skb);
                        if (msg)
                                br_do_suppress_nd(skb, br, vid, NULL, msg);
        }

        dest = eth_hdr(skb)->h_dest;
        if (is_broadcast_ether_addr(dest)) {
                br_flood(br, skb, BR_PKT_BROADCAST, false, true, vid);
        } else if (is_multicast_ether_addr(dest)) {
                if (unlikely(netpoll_tx_running(dev))) {
                        br_flood(br, skb, BR_PKT_MULTICAST, false, true, vid);
                        goto out;
                }
                if (br_multicast_rcv(&brmctx, &pmctx_null, vlan, skb, vid)) {
                        kfree_skb(skb);
                        goto out;
                }

                mdst = br_mdb_entry_skb_get(brmctx, skb, vid);
                if ((mdst || BR_INPUT_SKB_CB_MROUTERS_ONLY(skb)) &&
                    br_multicast_querier_exists(brmctx, eth_hdr(skb), mdst))
                        br_multicast_flood(mdst, skb, brmctx, false, true);
                else
                        br_flood(br, skb, BR_PKT_MULTICAST, false, true, vid);
        } else if ((dst = br_fdb_find_rcu(br, dest, vid)) != NULL) {
                br_forward(READ_ONCE(dst->dst), skb, false, true);
        } else {
                br_flood(br, skb, BR_PKT_UNICAST, false, true, vid);
        }
out:
        rcu_read_unlock();
        return NETDEV_TX_OK;
}

static int br_dev_init(struct net_device *dev)
{
        struct net_bridge *br = netdev_priv(dev);
        int err;

        err = br_fdb_hash_init(br);
        if (err)
                return err;

        err = br_mdb_hash_init(br);
        if (err) {
                br_fdb_hash_fini(br);
                return err;
        }

        err = br_vlan_init(br);
        if (err) {
                br_mdb_hash_fini(br);
                br_fdb_hash_fini(br);
                return err;
        }

        err = br_multicast_init_stats(br);
        if (err) {
                br_vlan_flush(br);
                br_mdb_hash_fini(br);
                br_fdb_hash_fini(br);
                return err;
        }

        netdev_lockdep_set_classes(dev);
        return 0;
}

static void br_dev_uninit(struct net_device *dev)
{
        struct net_bridge *br = netdev_priv(dev);

        br_multicast_dev_del(br);
        br_multicast_uninit_stats(br);
        br_vlan_flush(br);
        br_mdb_hash_fini(br);
        br_fdb_hash_fini(br);
}

static int br_dev_open(struct net_device *dev)
{
        struct net_bridge *br = netdev_priv(dev);

        netdev_update_features(dev);
        netif_start_queue(dev);
        br_stp_enable_bridge(br);
        br_multicast_open(br);

        if (br_opt_get(br, BROPT_MULTICAST_ENABLED))
                br_multicast_join_snoopers(br);

        return 0;
}

static void br_dev_set_multicast_list(struct net_device *dev)
{
}

static void br_dev_change_rx_flags(struct net_device *dev, int change)
{
        if (change & IFF_PROMISC)
                br_manage_promisc(netdev_priv(dev));
}

static int br_dev_stop(struct net_device *dev)
{
        struct net_bridge *br = netdev_priv(dev);

        br_stp_disable_bridge(br);
        br_multicast_stop(br);

        if (br_opt_get(br, BROPT_MULTICAST_ENABLED))
                br_multicast_leave_snoopers(br);

        netif_stop_queue(dev);

        return 0;
}

static int br_change_mtu(struct net_device *dev, int new_mtu)
{
        struct net_bridge *br = netdev_priv(dev);

        WRITE_ONCE(dev->mtu, new_mtu);

        /* this flag will be cleared if the MTU was automatically adjusted */
        br_opt_toggle(br, BROPT_MTU_SET_BY_USER, true);
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
        /* remember the MTU in the rtable for PMTU */
        dst_metric_set(&br->fake_rtable.dst, RTAX_MTU, new_mtu);
#endif

        return 0;
}

/* Allow setting mac address to any valid ethernet address. */
static int br_set_mac_address(struct net_device *dev, void *p)
{
        struct net_bridge *br = netdev_priv(dev);
        struct sockaddr *addr = p;

        if (!is_valid_ether_addr(addr->sa_data))
                return -EADDRNOTAVAIL;

        /* dev_set_mac_addr() can be called by a master device on bridge's
         * NETDEV_UNREGISTER, but since it's being destroyed do nothing
         */
        if (dev->reg_state != NETREG_REGISTERED)
                return -EBUSY;

        spin_lock_bh(&br->lock);
        if (!ether_addr_equal(dev->dev_addr, addr->sa_data)) {
                /* Mac address will be changed in br_stp_change_bridge_id(). */
                br_stp_change_bridge_id(br, addr->sa_data);
        }
        spin_unlock_bh(&br->lock);

        return 0;
}

static void br_getinfo(struct net_device *dev, struct ethtool_drvinfo *info)
{
        strscpy(info->driver, "bridge", sizeof(info->driver));
        strscpy(info->version, BR_VERSION, sizeof(info->version));
        strscpy(info->fw_version, "N/A", sizeof(info->fw_version));
        strscpy(info->bus_info, "N/A", sizeof(info->bus_info));
}

static int br_get_link_ksettings(struct net_device *dev,
                                 struct ethtool_link_ksettings *cmd)
{
        struct net_bridge *br = netdev_priv(dev);
        struct net_bridge_port *p;

        cmd->base.duplex = DUPLEX_UNKNOWN;
        cmd->base.port = PORT_OTHER;
        cmd->base.speed = SPEED_UNKNOWN;

        list_for_each_entry(p, &br->port_list, list) {
                struct ethtool_link_ksettings ecmd;
                struct net_device *pdev = p->dev;

                if (!netif_running(pdev) || !netif_oper_up(pdev))
                        continue;

                if (__ethtool_get_link_ksettings(pdev, &ecmd))
                        continue;

                if (ecmd.base.speed == (__u32)SPEED_UNKNOWN)
                        continue;

                if (cmd->base.speed == (__u32)SPEED_UNKNOWN ||
                    cmd->base.speed < ecmd.base.speed)
                        cmd->base.speed = ecmd.base.speed;
        }

        return 0;
}

static netdev_features_t br_fix_features(struct net_device *dev,
        netdev_features_t features)
{
        struct net_bridge *br = netdev_priv(dev);

        return br_features_recompute(br, features);
}

#ifdef CONFIG_NET_POLL_CONTROLLER
static void br_poll_controller(struct net_device *br_dev)
{
}

static void br_netpoll_cleanup(struct net_device *dev)
{
        struct net_bridge *br = netdev_priv(dev);
        struct net_bridge_port *p;

        list_for_each_entry(p, &br->port_list, list)
                br_netpoll_disable(p);
}

static int __br_netpoll_enable(struct net_bridge_port *p)
{
        struct netpoll *np;
        int err;

        np = kzalloc_obj(*p->np);
        if (!np)
                return -ENOMEM;

        err = __netpoll_setup(np, p->dev);
        if (err) {
                kfree(np);
                return err;
        }

        p->np = np;
        return err;
}

int br_netpoll_enable(struct net_bridge_port *p)
{
        if (!p->br->dev->npinfo)
                return 0;

        return __br_netpoll_enable(p);
}

static int br_netpoll_setup(struct net_device *dev)
{
        struct net_bridge *br = netdev_priv(dev);
        struct net_bridge_port *p;
        int err = 0;

        list_for_each_entry(p, &br->port_list, list) {
                if (!p->dev)
                        continue;
                err = __br_netpoll_enable(p);
                if (err)
                        goto fail;
        }

out:
        return err;

fail:
        br_netpoll_cleanup(dev);
        goto out;
}

void br_netpoll_disable(struct net_bridge_port *p)
{
        struct netpoll *np = p->np;

        if (!np)
                return;

        p->np = NULL;

        __netpoll_free(np);
}

#endif

static int br_add_slave(struct net_device *dev, struct net_device *slave_dev,
                        struct netlink_ext_ack *extack)

{
        struct net_bridge *br = netdev_priv(dev);

        return br_add_if(br, slave_dev, extack);
}

static int br_del_slave(struct net_device *dev, struct net_device *slave_dev)
{
        struct net_bridge *br = netdev_priv(dev);

        return br_del_if(br, slave_dev);
}

static int br_fill_forward_path(struct net_device_path_ctx *ctx,
                                struct net_device_path *path)
{
        struct net_bridge_fdb_entry *f;
        struct net_bridge_port *dst;
        struct net_bridge *br;

        if (netif_is_bridge_port(ctx->dev))
                return -1;

        br = netdev_priv(ctx->dev);

        br_vlan_fill_forward_path_pvid(br, ctx, path);

        f = br_fdb_find_rcu(br, ctx->daddr, path->bridge.vlan_id);
        if (!f)
                return -1;

        dst = READ_ONCE(f->dst);
        if (!dst)
                return -1;

        if (br_vlan_fill_forward_path_mode(br, dst, path))
                return -1;

        path->type = DEV_PATH_BRIDGE;
        path->dev = dst->br->dev;
        ctx->dev = dst->dev;

        switch (path->bridge.vlan_mode) {
        case DEV_PATH_BR_VLAN_TAG:
                if (ctx->num_vlans >= ARRAY_SIZE(ctx->vlan))
                        return -ENOSPC;
                ctx->vlan[ctx->num_vlans].id = path->bridge.vlan_id;
                ctx->vlan[ctx->num_vlans].proto = path->bridge.vlan_proto;
                ctx->num_vlans++;
                break;
        case DEV_PATH_BR_VLAN_UNTAG_HW:
        case DEV_PATH_BR_VLAN_UNTAG:
                ctx->num_vlans--;
                break;
        case DEV_PATH_BR_VLAN_KEEP:
                break;
        }

        return 0;
}

static const struct ethtool_ops br_ethtool_ops = {
        .get_drvinfo                 = br_getinfo,
        .get_link                 = ethtool_op_get_link,
        .get_link_ksettings         = br_get_link_ksettings,
};

static const struct net_device_ops br_netdev_ops = {
        .ndo_open                 = br_dev_open,
        .ndo_stop                 = br_dev_stop,
        .ndo_init                 = br_dev_init,
        .ndo_uninit                 = br_dev_uninit,
        .ndo_start_xmit                 = br_dev_xmit,
        .ndo_get_stats64         = dev_get_tstats64,
        .ndo_set_mac_address         = br_set_mac_address,
        .ndo_set_rx_mode         = br_dev_set_multicast_list,
        .ndo_change_rx_flags         = br_dev_change_rx_flags,
        .ndo_change_mtu                 = br_change_mtu,
        .ndo_siocdevprivate         = br_dev_siocdevprivate,
#ifdef CONFIG_NET_POLL_CONTROLLER
        .ndo_netpoll_setup         = br_netpoll_setup,
        .ndo_netpoll_cleanup         = br_netpoll_cleanup,
        .ndo_poll_controller         = br_poll_controller,
#endif
        .ndo_add_slave                 = br_add_slave,
        .ndo_del_slave                 = br_del_slave,
        .ndo_fix_features        = br_fix_features,
        .ndo_fdb_add                 = br_fdb_add,
        .ndo_fdb_del                 = br_fdb_delete,
        .ndo_fdb_del_bulk         = br_fdb_delete_bulk,
        .ndo_fdb_dump                 = br_fdb_dump,
        .ndo_fdb_get                 = br_fdb_get,
        .ndo_mdb_add                 = br_mdb_add,
        .ndo_mdb_del                 = br_mdb_del,
        .ndo_mdb_del_bulk         = br_mdb_del_bulk,
        .ndo_mdb_dump                 = br_mdb_dump,
        .ndo_mdb_get                 = br_mdb_get,
        .ndo_bridge_getlink         = br_getlink,
        .ndo_bridge_setlink         = br_setlink,
        .ndo_bridge_dellink         = br_dellink,
        .ndo_features_check         = passthru_features_check,
        .ndo_fill_forward_path         = br_fill_forward_path,
};

static const struct device_type br_type = {
        .name        = "bridge",
};

void br_dev_setup(struct net_device *dev)
{
        struct net_bridge *br = netdev_priv(dev);

        eth_hw_addr_random(dev);
        ether_setup(dev);

        dev->netdev_ops = &br_netdev_ops;
        dev->needs_free_netdev = true;
        dev->ethtool_ops = &br_ethtool_ops;
        SET_NETDEV_DEVTYPE(dev, &br_type);
        dev->priv_flags = IFF_EBRIDGE | IFF_NO_QUEUE;
        dev->lltx = true;
        dev->netns_immutable = true;

        dev->features = COMMON_FEATURES | NETIF_F_HW_VLAN_CTAG_TX |
                        NETIF_F_HW_VLAN_STAG_TX;
        dev->hw_features = COMMON_FEATURES | NETIF_F_HW_VLAN_CTAG_TX |
                           NETIF_F_HW_VLAN_STAG_TX;
        dev->vlan_features = COMMON_FEATURES;
        dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;

        br->dev = dev;
        spin_lock_init(&br->lock);
        INIT_LIST_HEAD(&br->port_list);
        INIT_HLIST_HEAD(&br->fdb_list);
        INIT_HLIST_HEAD(&br->frame_type_list);
#if IS_ENABLED(CONFIG_BRIDGE_MRP)
        INIT_HLIST_HEAD(&br->mrp_list);
#endif
#if IS_ENABLED(CONFIG_BRIDGE_CFM)
        INIT_HLIST_HEAD(&br->mep_list);
#endif
        spin_lock_init(&br->hash_lock);

        br->bridge_id.prio[0] = 0x80;
        br->bridge_id.prio[1] = 0x00;

        ether_addr_copy(br->group_addr, eth_stp_addr);

        br->stp_enabled = BR_NO_STP;
        br->stp_mode = BR_STP_MODE_AUTO;
        br->group_fwd_mask = BR_GROUPFWD_DEFAULT;
        br->group_fwd_mask_required = BR_GROUPFWD_DEFAULT;

        br->designated_root = br->bridge_id;
        br->bridge_max_age = br->max_age = 20 * HZ;
        br->bridge_hello_time = br->hello_time = 2 * HZ;
        br->bridge_forward_delay = br->forward_delay = 15 * HZ;
        br->bridge_ageing_time = br->ageing_time = BR_DEFAULT_AGEING_TIME;
        dev->max_mtu = ETH_MAX_MTU;

        br_netfilter_rtable_init(br);
        br_stp_timer_init(br);
        br_multicast_init(br);
        INIT_DELAYED_WORK(&br->gc_work, br_fdb_cleanup);
}

























































































































    2 













































    2 








    2 













    2 





































    2 







    2 














































































































































































    2 



    2 





































































































































































































































































































































































  161 







  161 

















   19 












   19 











  161 








  161 





















































































































































































   11 








   11 








  324 







   78 



   14 
   74 









































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Cryptographic API for algorithms (i.e., low-level API).
 *
 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 */

#include <crypto/algapi.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/fips.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/rtnetlink.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/workqueue.h>

#include "internal.h"

static LIST_HEAD(crypto_template_list);

static inline void crypto_check_module_sig(struct module *mod)
{
        if (fips_enabled && mod && !module_sig_ok(mod))
                panic("Module %s signature verification failed in FIPS mode\n",
                      module_name(mod));
}

static int crypto_check_alg(struct crypto_alg *alg)
{
        crypto_check_module_sig(alg->cra_module);

        if (!alg->cra_name[0] || !alg->cra_driver_name[0])
                return -EINVAL;

        if (alg->cra_alignmask & (alg->cra_alignmask + 1))
                return -EINVAL;

        /* General maximums for all algs. */
        if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK)
                return -EINVAL;

        if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE)
                return -EINVAL;

        /* Lower maximums for specific alg types. */
        if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
                               CRYPTO_ALG_TYPE_CIPHER) {
                if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)
                        return -EINVAL;

                if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE)
                        return -EINVAL;
        }

        if (alg->cra_priority < 0)
                return -EINVAL;

        refcount_set(&alg->cra_refcnt, 1);

        return 0;
}

static void crypto_free_instance(struct crypto_instance *inst)
{
        inst->alg.cra_type->free(inst);
}

static void crypto_destroy_instance_workfn(struct work_struct *w)
{
        struct crypto_template *tmpl = container_of(w, struct crypto_template,
                                                    free_work);
        struct crypto_instance *inst;
        struct hlist_node *n;
        HLIST_HEAD(list);

        down_write(&crypto_alg_sem);
        hlist_for_each_entry_safe(inst, n, &tmpl->dead, list) {
                if (refcount_read(&inst->alg.cra_refcnt) != -1)
                        continue;
                hlist_del(&inst->list);
                hlist_add_head(&inst->list, &list);
        }
        up_write(&crypto_alg_sem);

        hlist_for_each_entry_safe(inst, n, &list, list)
                crypto_free_instance(inst);
}

static void crypto_destroy_instance(struct crypto_alg *alg)
{
        struct crypto_instance *inst = container_of(alg,
                                                    struct crypto_instance,
                                                    alg);
        struct crypto_template *tmpl = inst->tmpl;

        refcount_set(&alg->cra_refcnt, -1);
        schedule_work(&tmpl->free_work);
}

/*
 * This function adds a spawn to the list secondary_spawns which
 * will be used at the end of crypto_remove_spawns to unregister
 * instances, unless the spawn happens to be one that is depended
 * on by the new algorithm (nalg in crypto_remove_spawns).
 *
 * This function is also responsible for resurrecting any algorithms
 * in the dependency chain of nalg by unsetting n->dead.
 */
static struct list_head *crypto_more_spawns(struct crypto_alg *alg,
                                            struct list_head *stack,
                                            struct list_head *top,
                                            struct list_head *secondary_spawns)
{
        struct crypto_spawn *spawn, *n;

        spawn = list_first_entry_or_null(stack, struct crypto_spawn, list);
        if (!spawn)
                return NULL;

        n = list_prev_entry(spawn, list);
        list_move(&spawn->list, secondary_spawns);

        if (list_is_last(&n->list, stack))
                return top;

        n = list_next_entry(n, list);
        if (!spawn->dead)
                n->dead = false;

        return &n->inst->alg.cra_users;
}

static void crypto_remove_instance(struct crypto_instance *inst,
                                   struct list_head *list)
{
        struct crypto_template *tmpl = inst->tmpl;

        if (crypto_is_dead(&inst->alg))
                return;

        inst->alg.cra_flags |= CRYPTO_ALG_DEAD;

        if (!tmpl)
                return;

        list_del_init(&inst->alg.cra_list);
        hlist_del(&inst->list);
        hlist_add_head(&inst->list, &tmpl->dead);

        BUG_ON(!list_empty(&inst->alg.cra_users));

        crypto_alg_put(&inst->alg);
}

/*
 * Given an algorithm alg, remove all algorithms that depend on it
 * through spawns.  If nalg is not null, then exempt any algorithms
 * that is depended on by nalg.  This is useful when nalg itself
 * depends on alg.
 */
void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
                          struct crypto_alg *nalg)
{
        u32 new_type = (nalg ?: alg)->cra_flags;
        struct crypto_spawn *spawn, *n;
        LIST_HEAD(secondary_spawns);
        struct list_head *spawns;
        LIST_HEAD(stack);
        LIST_HEAD(top);

        spawns = &alg->cra_users;
        list_for_each_entry_safe(spawn, n, spawns, list) {
                if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)
                        continue;

                list_move(&spawn->list, &top);
        }

        /*
         * Perform a depth-first walk starting from alg through
         * the cra_users tree.  The list stack records the path
         * from alg to the current spawn.
         */
        spawns = &top;
        do {
                while (!list_empty(spawns)) {
                        struct crypto_instance *inst;

                        spawn = list_first_entry(spawns, struct crypto_spawn,
                                                 list);
                        inst = spawn->inst;

                        list_move(&spawn->list, &stack);
                        spawn->dead = !spawn->registered || &inst->alg != nalg;

                        if (!spawn->registered)
                                break;

                        BUG_ON(&inst->alg == alg);

                        if (&inst->alg == nalg)
                                break;

                        spawns = &inst->alg.cra_users;

                        /*
                         * Even if spawn->registered is true, the
                         * instance itself may still be unregistered.
                         * This is because it may have failed during
                         * registration.  Therefore we still need to
                         * make the following test.
                         *
                         * We may encounter an unregistered instance here, since
                         * an instance's spawns are set up prior to the instance
                         * being registered.  An unregistered instance will have
                         * NULL ->cra_users.next, since ->cra_users isn't
                         * properly initialized until registration.  But an
                         * unregistered instance cannot have any users, so treat
                         * it the same as ->cra_users being empty.
                         */
                        if (spawns->next == NULL)
                                break;
                }
        } while ((spawns = crypto_more_spawns(alg, &stack, &top,
                                              &secondary_spawns)));

        /*
         * Remove all instances that are marked as dead.  Also
         * complete the resurrection of the others by moving them
         * back to the cra_users list.
         */
        list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {
                if (!spawn->dead)
                        list_move(&spawn->list, &spawn->alg->cra_users);
                else if (spawn->registered)
                        crypto_remove_instance(spawn->inst, list);
        }
}
EXPORT_SYMBOL_GPL(crypto_remove_spawns);

static void crypto_alg_finish_registration(struct crypto_alg *alg,
                                           struct list_head *algs_to_put)
        __must_hold(&crypto_alg_sem)
{
        struct crypto_alg *q;

        list_for_each_entry(q, &crypto_alg_list, cra_list) {
                if (q == alg)
                        continue;

                if (crypto_is_moribund(q))
                        continue;

                if (crypto_is_larval(q))
                        continue;

                if (strcmp(alg->cra_name, q->cra_name))
                        continue;

                if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&
                    q->cra_priority > alg->cra_priority)
                        continue;

                crypto_remove_spawns(q, algs_to_put, alg);
        }

        crypto_notify(CRYPTO_MSG_ALG_LOADED, alg);
}

static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)
{
        struct crypto_larval *larval;

        if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) ||
            (alg->cra_flags & CRYPTO_ALG_INTERNAL))
                return NULL; /* No self-test needed */

        larval = crypto_larval_alloc(alg->cra_name,
                                     alg->cra_flags | CRYPTO_ALG_TESTED, 0);
        if (IS_ERR(larval))
                return larval;

        larval->adult = crypto_mod_get(alg);
        if (!larval->adult) {
                kfree(larval);
                return ERR_PTR(-ENOENT);
        }

        refcount_set(&larval->alg.cra_refcnt, 1);
        memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,
               CRYPTO_MAX_ALG_NAME);
        larval->alg.cra_priority = alg->cra_priority;

        return larval;
}

static struct crypto_larval *
__crypto_register_alg(struct crypto_alg *alg, struct list_head *algs_to_put)
        __must_hold(&crypto_alg_sem)
{
        struct crypto_alg *q;
        struct crypto_larval *larval;
        int ret = -EAGAIN;

        if (crypto_is_dead(alg))
                goto err;

        INIT_LIST_HEAD(&alg->cra_users);

        ret = -EEXIST;

        list_for_each_entry(q, &crypto_alg_list, cra_list) {
                if (q == alg)
                        goto err;

                if (crypto_is_moribund(q))
                        continue;

                if (crypto_is_larval(q)) {
                        if (!strcmp(alg->cra_driver_name, q->cra_driver_name))
                                goto err;
                        continue;
                }

                if (!strcmp(q->cra_driver_name, alg->cra_name) ||
                    !strcmp(q->cra_driver_name, alg->cra_driver_name) ||
                    !strcmp(q->cra_name, alg->cra_driver_name))
                        goto err;
        }

        larval = crypto_alloc_test_larval(alg);
        if (IS_ERR(larval))
                goto out;

        list_add(&alg->cra_list, &crypto_alg_list);

        if (larval) {
                /* No cheating! */
                alg->cra_flags &= ~CRYPTO_ALG_TESTED;

                list_add(&larval->alg.cra_list, &crypto_alg_list);
        } else {
                alg->cra_flags |= CRYPTO_ALG_TESTED;
                crypto_alg_finish_registration(alg, algs_to_put);
        }

out:
        return larval;

err:
        larval = ERR_PTR(ret);
        goto out;
}

void crypto_alg_tested(const char *name, int err)
{
        struct crypto_larval *test;
        struct crypto_alg *alg;
        struct crypto_alg *q;
        LIST_HEAD(list);

        down_write(&crypto_alg_sem);
        list_for_each_entry(q, &crypto_alg_list, cra_list) {
                if (crypto_is_moribund(q) || !crypto_is_larval(q))
                        continue;

                test = (struct crypto_larval *)q;

                if (!strcmp(q->cra_driver_name, name))
                        goto found;
        }

        pr_err("alg: Unexpected test result for %s: %d\n", name, err);
        up_write(&crypto_alg_sem);
        return;

found:
        q->cra_flags |= CRYPTO_ALG_DEAD;
        alg = test->adult;

        if (crypto_is_dead(alg))
                goto complete;

        if (err == -ECANCELED)
                alg->cra_flags |= CRYPTO_ALG_FIPS_INTERNAL;
        else if (err)
                goto complete;
        else
                alg->cra_flags &= ~CRYPTO_ALG_FIPS_INTERNAL;

        alg->cra_flags |= CRYPTO_ALG_TESTED;

        crypto_alg_finish_registration(alg, &list);

complete:
        list_del_init(&test->alg.cra_list);
        complete_all(&test->completion);

        up_write(&crypto_alg_sem);

        crypto_alg_put(&test->alg);
        crypto_remove_final(&list);
}
EXPORT_SYMBOL_GPL(crypto_alg_tested);

void crypto_remove_final(struct list_head *list)
{
        struct crypto_alg *alg;
        struct crypto_alg *n;

        list_for_each_entry_safe(alg, n, list, cra_list) {
                list_del_init(&alg->cra_list);
                crypto_alg_put(alg);
        }
}
EXPORT_SYMBOL_GPL(crypto_remove_final);

static void crypto_free_alg(struct crypto_alg *alg)
{
        unsigned int algsize = alg->cra_type->algsize;
        u8 *p = (u8 *)alg - algsize;

        crypto_destroy_alg(alg);
        kfree(p);
}

int crypto_register_alg(struct crypto_alg *alg)
{
        struct crypto_larval *larval;
        bool test_started = false;
        LIST_HEAD(algs_to_put);
        int err;

        alg->cra_flags &= ~CRYPTO_ALG_DEAD;
        err = crypto_check_alg(alg);
        if (err)
                return err;

        if (alg->cra_flags & CRYPTO_ALG_DUP_FIRST &&
            !WARN_ON_ONCE(alg->cra_destroy)) {
                unsigned int algsize = alg->cra_type->algsize;
                u8 *p = (u8 *)alg - algsize;

                p = kmemdup(p, algsize + sizeof(*alg), GFP_KERNEL);
                if (!p)
                        return -ENOMEM;

                alg = (void *)(p + algsize);
                alg->cra_destroy = crypto_free_alg;
        }

        down_write(&crypto_alg_sem);
        larval = __crypto_register_alg(alg, &algs_to_put);
        if (!IS_ERR_OR_NULL(larval)) {
                test_started = crypto_boot_test_finished();
                larval->test_started = test_started;
        }
        up_write(&crypto_alg_sem);

        if (IS_ERR(larval)) {
                crypto_alg_put(alg);
                return PTR_ERR(larval);
        }

        if (test_started)
                crypto_schedule_test(larval);
        else
                crypto_remove_final(&algs_to_put);

        return 0;
}
EXPORT_SYMBOL_GPL(crypto_register_alg);

static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
{
        if (unlikely(list_empty(&alg->cra_list)))
                return -ENOENT;

        alg->cra_flags |= CRYPTO_ALG_DEAD;

        list_del_init(&alg->cra_list);
        crypto_remove_spawns(alg, list, NULL);

        return 0;
}

void crypto_unregister_alg(struct crypto_alg *alg)
{
        int ret;
        LIST_HEAD(list);

        down_write(&crypto_alg_sem);
        ret = crypto_remove_alg(alg, &list);
        up_write(&crypto_alg_sem);

        if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name))
                return;

        WARN_ON(!alg->cra_destroy && refcount_read(&alg->cra_refcnt) != 1);

        list_add(&alg->cra_list, &list);
        crypto_remove_final(&list);
}
EXPORT_SYMBOL_GPL(crypto_unregister_alg);

int crypto_register_algs(struct crypto_alg *algs, int count)
{
        int i, ret;

        for (i = 0; i < count; i++) {
                ret = crypto_register_alg(&algs[i]);
                if (ret) {
                        crypto_unregister_algs(algs, i);
                        return ret;
                }
        }

        return 0;
}
EXPORT_SYMBOL_GPL(crypto_register_algs);

void crypto_unregister_algs(struct crypto_alg *algs, int count)
{
        int i;

        for (i = count - 1; i >= 0; --i)
                crypto_unregister_alg(&algs[i]);
}
EXPORT_SYMBOL_GPL(crypto_unregister_algs);

int crypto_register_template(struct crypto_template *tmpl)
{
        struct crypto_template *q;
        int err = -EEXIST;

        INIT_WORK(&tmpl->free_work, crypto_destroy_instance_workfn);

        down_write(&crypto_alg_sem);

        crypto_check_module_sig(tmpl->module);

        list_for_each_entry(q, &crypto_template_list, list) {
                if (q == tmpl)
                        goto out;
        }

        list_add(&tmpl->list, &crypto_template_list);
        err = 0;
out:
        up_write(&crypto_alg_sem);
        return err;
}
EXPORT_SYMBOL_GPL(crypto_register_template);

int crypto_register_templates(struct crypto_template *tmpls, int count)
{
        int i, err;

        for (i = 0; i < count; i++) {
                err = crypto_register_template(&tmpls[i]);
                if (err)
                        goto out;
        }
        return 0;

out:
        for (--i; i >= 0; --i)
                crypto_unregister_template(&tmpls[i]);
        return err;
}
EXPORT_SYMBOL_GPL(crypto_register_templates);

void crypto_unregister_template(struct crypto_template *tmpl)
{
        struct crypto_instance *inst;
        struct hlist_node *n;
        struct hlist_head *list;
        LIST_HEAD(users);

        down_write(&crypto_alg_sem);

        BUG_ON(list_empty(&tmpl->list));
        list_del_init(&tmpl->list);

        list = &tmpl->instances;
        hlist_for_each_entry(inst, list, list) {
                int err = crypto_remove_alg(&inst->alg, &users);

                BUG_ON(err);
        }

        up_write(&crypto_alg_sem);

        hlist_for_each_entry_safe(inst, n, list, list) {
                BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1);
                crypto_free_instance(inst);
        }
        crypto_remove_final(&users);

        flush_work(&tmpl->free_work);
}
EXPORT_SYMBOL_GPL(crypto_unregister_template);

void crypto_unregister_templates(struct crypto_template *tmpls, int count)
{
        int i;

        for (i = count - 1; i >= 0; --i)
                crypto_unregister_template(&tmpls[i]);
}
EXPORT_SYMBOL_GPL(crypto_unregister_templates);

static struct crypto_template *__crypto_lookup_template(const char *name)
{
        struct crypto_template *q, *tmpl = NULL;

        down_read(&crypto_alg_sem);
        list_for_each_entry(q, &crypto_template_list, list) {
                if (strcmp(q->name, name))
                        continue;
                if (unlikely(!crypto_tmpl_get(q)))
                        continue;

                tmpl = q;
                break;
        }
        up_read(&crypto_alg_sem);

        return tmpl;
}

struct crypto_template *crypto_lookup_template(const char *name)
{
        return try_then_request_module(__crypto_lookup_template(name),
                                       "crypto-%s", name);
}
EXPORT_SYMBOL_GPL(crypto_lookup_template);

int crypto_register_instance(struct crypto_template *tmpl,
                             struct crypto_instance *inst)
{
        struct crypto_larval *larval;
        struct crypto_spawn *spawn;
        u32 fips_internal = 0;
        LIST_HEAD(algs_to_put);
        int err;

        err = crypto_check_alg(&inst->alg);
        if (err)
                return err;

        inst->alg.cra_module = tmpl->module;
        inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;
        inst->alg.cra_destroy = crypto_destroy_instance;

        down_write(&crypto_alg_sem);

        larval = ERR_PTR(-EAGAIN);
        for (spawn = inst->spawns; spawn;) {
                struct crypto_spawn *next;

                if (spawn->dead)
                        goto unlock;

                next = spawn->next;
                spawn->inst = inst;
                spawn->registered = true;

                fips_internal |= spawn->alg->cra_flags;

                crypto_mod_put(spawn->alg);

                spawn = next;
        }

        inst->alg.cra_flags |= (fips_internal & CRYPTO_ALG_FIPS_INTERNAL);

        larval = __crypto_register_alg(&inst->alg, &algs_to_put);
        if (IS_ERR(larval))
                goto unlock;
        else if (larval)
                larval->test_started = true;

        hlist_add_head(&inst->list, &tmpl->instances);
        inst->tmpl = tmpl;

unlock:
        up_write(&crypto_alg_sem);

        if (IS_ERR(larval))
                return PTR_ERR(larval);

        if (larval)
                crypto_schedule_test(larval);
        else
                crypto_remove_final(&algs_to_put);

        return 0;
}
EXPORT_SYMBOL_GPL(crypto_register_instance);

void crypto_unregister_instance(struct crypto_instance *inst)
{
        LIST_HEAD(list);

        down_write(&crypto_alg_sem);

        crypto_remove_spawns(&inst->alg, &list, NULL);
        crypto_remove_instance(inst, &list);

        up_write(&crypto_alg_sem);

        crypto_remove_final(&list);
}
EXPORT_SYMBOL_GPL(crypto_unregister_instance);

int crypto_grab_spawn(struct crypto_spawn *spawn, struct crypto_instance *inst,
                      const char *name, u32 type, u32 mask)
{
        struct crypto_alg *alg;
        int err = -EAGAIN;

        if (WARN_ON_ONCE(inst == NULL))
                return -EINVAL;

        /* Allow the result of crypto_attr_alg_name() to be passed directly */
        if (IS_ERR(name))
                return PTR_ERR(name);

        alg = crypto_find_alg(name, spawn->frontend,
                              type | CRYPTO_ALG_FIPS_INTERNAL, mask);
        if (IS_ERR(alg))
                return PTR_ERR(alg);

        down_write(&crypto_alg_sem);
        if (!crypto_is_moribund(alg)) {
                list_add(&spawn->list, &alg->cra_users);
                spawn->alg = alg;
                spawn->mask = mask;
                spawn->next = inst->spawns;
                inst->spawns = spawn;
                inst->alg.cra_flags |=
                        (alg->cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
                err = 0;
        }
        up_write(&crypto_alg_sem);
        if (err)
                crypto_mod_put(alg);
        return err;
}
EXPORT_SYMBOL_GPL(crypto_grab_spawn);

void crypto_drop_spawn(struct crypto_spawn *spawn)
{
        if (!spawn->alg) /* not yet initialized? */
                return;

        down_write(&crypto_alg_sem);
        if (!spawn->dead)
                list_del(&spawn->list);
        up_write(&crypto_alg_sem);

        if (!spawn->registered)
                crypto_mod_put(spawn->alg);
}
EXPORT_SYMBOL_GPL(crypto_drop_spawn);

static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn)
{
        struct crypto_alg *alg = ERR_PTR(-EAGAIN);
        struct crypto_alg *target;
        bool shoot = false;

        down_read(&crypto_alg_sem);
        if (!spawn->dead) {
                alg = spawn->alg;
                if (!crypto_mod_get(alg)) {
                        target = crypto_alg_get(alg);
                        shoot = true;
                        alg = ERR_PTR(-EAGAIN);
                }
        }
        up_read(&crypto_alg_sem);

        if (shoot) {
                crypto_shoot_alg(target);
                crypto_alg_put(target);
        }

        return alg;
}

struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type,
                                    u32 mask)
{
        struct crypto_alg *alg;
        struct crypto_tfm *tfm;

        alg = crypto_spawn_alg(spawn);
        if (IS_ERR(alg))
                return ERR_CAST(alg);

        tfm = ERR_PTR(-EINVAL);
        if (unlikely((alg->cra_flags ^ type) & mask))
                goto out_put_alg;

        tfm = __crypto_alloc_tfm(alg, type, mask);
        if (IS_ERR(tfm))
                goto out_put_alg;

        return tfm;

out_put_alg:
        crypto_mod_put(alg);
        return tfm;
}
EXPORT_SYMBOL_GPL(crypto_spawn_tfm);

void *crypto_spawn_tfm2(struct crypto_spawn *spawn)
{
        struct crypto_alg *alg;
        struct crypto_tfm *tfm;

        alg = crypto_spawn_alg(spawn);
        if (IS_ERR(alg))
                return ERR_CAST(alg);

        tfm = crypto_create_tfm(alg, spawn->frontend);
        if (IS_ERR(tfm))
                goto out_put_alg;

        return tfm;

out_put_alg:
        crypto_mod_put(alg);
        return tfm;
}
EXPORT_SYMBOL_GPL(crypto_spawn_tfm2);

int crypto_register_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&crypto_chain, nb);
}
EXPORT_SYMBOL_GPL(crypto_register_notifier);

int crypto_unregister_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&crypto_chain, nb);
}
EXPORT_SYMBOL_GPL(crypto_unregister_notifier);

struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb)
{
        struct rtattr *rta = tb[0];
        struct crypto_attr_type *algt;

        if (!rta)
                return ERR_PTR(-ENOENT);
        if (RTA_PAYLOAD(rta) < sizeof(*algt))
                return ERR_PTR(-EINVAL);
        if (rta->rta_type != CRYPTOA_TYPE)
                return ERR_PTR(-EINVAL);

        algt = RTA_DATA(rta);

        return algt;
}
EXPORT_SYMBOL_GPL(crypto_get_attr_type);

/**
 * crypto_check_attr_type() - check algorithm type and compute inherited mask
 * @tb: the template parameters
 * @type: the algorithm type the template would be instantiated as
 * @mask_ret: (output) the mask that should be passed to crypto_grab_*()
 *              to restrict the flags of any inner algorithms
 *
 * Validate that the algorithm type the user requested is compatible with the
 * one the template would actually be instantiated as.  E.g., if the user is
 * doing crypto_alloc_shash("cbc(aes)", ...), this would return an error because
 * the "cbc" template creates an "skcipher" algorithm, not an "shash" algorithm.
 *
 * Also compute the mask to use to restrict the flags of any inner algorithms.
 *
 * Return: 0 on success; -errno on failure
 */
int crypto_check_attr_type(struct rtattr **tb, u32 type, u32 *mask_ret)
{
        struct crypto_attr_type *algt;

        algt = crypto_get_attr_type(tb);
        if (IS_ERR(algt))
                return PTR_ERR(algt);

        if ((algt->type ^ type) & algt->mask)
                return -EINVAL;

        *mask_ret = crypto_algt_inherited_mask(algt);
        return 0;
}
EXPORT_SYMBOL_GPL(crypto_check_attr_type);

const char *crypto_attr_alg_name(struct rtattr *rta)
{
        struct crypto_attr_alg *alga;

        if (!rta)
                return ERR_PTR(-ENOENT);
        if (RTA_PAYLOAD(rta) < sizeof(*alga))
                return ERR_PTR(-EINVAL);
        if (rta->rta_type != CRYPTOA_ALG)
                return ERR_PTR(-EINVAL);

        alga = RTA_DATA(rta);
        alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0;

        return alga->name;
}
EXPORT_SYMBOL_GPL(crypto_attr_alg_name);

int __crypto_inst_setname(struct crypto_instance *inst, const char *name,
                          const char *driver, struct crypto_alg *alg)
{
        if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,
                     alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
                return -ENAMETOOLONG;

        if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
                     driver, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
                return -ENAMETOOLONG;

        return 0;
}
EXPORT_SYMBOL_GPL(__crypto_inst_setname);

void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen)
{
        INIT_LIST_HEAD(&queue->list);
        queue->backlog = &queue->list;
        queue->qlen = 0;
        queue->max_qlen = max_qlen;
}
EXPORT_SYMBOL_GPL(crypto_init_queue);

int crypto_enqueue_request(struct crypto_queue *queue,
                           struct crypto_async_request *request)
{
        int err = -EINPROGRESS;

        if (unlikely(queue->qlen >= queue->max_qlen)) {
                if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
                        err = -ENOSPC;
                        goto out;
                }
                err = -EBUSY;
                if (queue->backlog == &queue->list)
                        queue->backlog = &request->list;
        }

        queue->qlen++;
        list_add_tail(&request->list, &queue->list);

out:
        return err;
}
EXPORT_SYMBOL_GPL(crypto_enqueue_request);

void crypto_enqueue_request_head(struct crypto_queue *queue,
                                 struct crypto_async_request *request)
{
        if (unlikely(queue->qlen >= queue->max_qlen))
                queue->backlog = queue->backlog->prev;

        queue->qlen++;
        list_add(&request->list, &queue->list);
}
EXPORT_SYMBOL_GPL(crypto_enqueue_request_head);

struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue)
{
        struct list_head *request;

        if (unlikely(!queue->qlen))
                return NULL;

        queue->qlen--;

        if (queue->backlog != &queue->list)
                queue->backlog = queue->backlog->next;

        request = queue->list.next;
        list_del_init(request);

        return list_entry(request, struct crypto_async_request, list);
}
EXPORT_SYMBOL_GPL(crypto_dequeue_request);

static inline void crypto_inc_byte(u8 *a, unsigned int size)
{
        u8 *b = (a + size);
        u8 c;

        for (; size; size--) {
                c = *--b + 1;
                *b = c;
                if (c)
                        break;
        }
}

void crypto_inc(u8 *a, unsigned int size)
{
        __be32 *b = (__be32 *)(a + size);
        u32 c;

        if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
            IS_ALIGNED((unsigned long)b, __alignof__(*b)))
                for (; size >= 4; size -= 4) {
                        c = be32_to_cpu(*--b) + 1;
                        *b = cpu_to_be32(c);
                        if (likely(c))
                                return;
                }

        crypto_inc_byte(a, size);
}
EXPORT_SYMBOL_GPL(crypto_inc);

unsigned int crypto_alg_extsize(struct crypto_alg *alg)
{
        return alg->cra_ctxsize +
               (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
}
EXPORT_SYMBOL_GPL(crypto_alg_extsize);

int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
                        u32 type, u32 mask)
{
        int ret = 0;
        struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask);

        if (!IS_ERR(alg)) {
                crypto_mod_put(alg);
                ret = 1;
        }

        return ret;
}
EXPORT_SYMBOL_GPL(crypto_type_has_alg);

static void __init crypto_start_tests(void)
{
        if (!IS_BUILTIN(CONFIG_CRYPTO_ALGAPI))
                return;

        if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS))
                return;

        set_crypto_boot_test_finished();

        for (;;) {
                struct crypto_larval *larval = NULL;
                struct crypto_alg *q;

                down_write(&crypto_alg_sem);

                list_for_each_entry(q, &crypto_alg_list, cra_list) {
                        struct crypto_larval *l;

                        if (!crypto_is_larval(q))
                                continue;

                        l = (void *)q;

                        if (!crypto_is_test_larval(l))
                                continue;

                        if (l->test_started)
                                continue;

                        l->test_started = true;
                        larval = l;
                        break;
                }

                up_write(&crypto_alg_sem);

                if (!larval)
                        break;

                crypto_schedule_test(larval);
        }
}

static int __init crypto_algapi_init(void)
{
        crypto_init_proc();
        crypto_start_tests();
        return 0;
}

static void __exit crypto_algapi_exit(void)
{
        crypto_exit_proc();
}

/*
 * We run this at late_initcall so that all the built-in algorithms
 * have had a chance to register themselves first.
 */
late_initcall(crypto_algapi_init);
module_exit(crypto_algapi_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Cryptographic algorithms API");
MODULE_SOFTDEP("pre: cryptomgr");






































































































































































    1 





    1 















    8 











    6 
    5 




    4 




















   17 


   17 











   17 





   26 


   20 









   11 




























































































































    3 












    3 
    2 


































    8 





















    8 

    8 








    8 









    8 








    3 











    1 






    1 


































































    3 




    4 





    1 


























    8 
    5 
    4 


    8 























































    2 
   10 
    1 




















  287 

















   17 







   17 
   17 
   17 









   17 























   14 





    4 










    9 



    8 








    8 


    8 




    9 
   17 




































































    8 





    5 
    2 






    3 
    3 
    3 




































    3 




    1 


    2 


    1 


    1 



















    2 














   11 
    5 









   11 




   11 




   11 











































   11 






    8 
    8 

    8 




    8 




































































































































































    6 


    1 


    1 


    5 








    6 




    6 


    6 
    3 




    6 







    2 










    2 

    2 
    1 
    1 

    1 

    1 




    1 


































   50 



   50 
   50 
   18 












   13 








   13 

   13 





   13 


   12 
   12 




    1 





    1 



    1 







    9 








    1 


    8 











































   18 





    2 


    2 




    1 







    8 







    5 


    1 



    1 

    3 



    2 




    5 

    2 



    2 









    8 
    1 























    4 




    2 







    2 
























    7 





    2 



    7 















    5 



    2 









   13 








   13 



    6 

    1 





    6 



   11 





    6 






    6 












































    7 




    7 



    4 



    7 














   21 












   21 
   21 
    1 

    7 






   21 







   25 

































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
// SPDX-License-Identifier: GPL-2.0
/*
 * Management Component Transport Protocol (MCTP) - routing
 * implementation.
 *
 * This is currently based on a simple routing table, with no dst cache. The
 * number of routes should stay fairly small, so the lookup cost is small.
 *
 * Copyright (c) 2021 Code Construct
 * Copyright (c) 2021 Google
 */

#include <linux/idr.h>
#include <linux/kconfig.h>
#include <linux/mctp.h>
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/skbuff.h>

#include <kunit/static_stub.h>

#include <uapi/linux/if_arp.h>

#include <net/mctp.h>
#include <net/mctpdevice.h>
#include <net/netlink.h>
#include <net/sock.h>

#include <trace/events/mctp.h>

static const unsigned int mctp_message_maxlen = 64 * 1024;
static const unsigned long mctp_key_lifetime = 6 * CONFIG_HZ;

static void mctp_flow_prepare_output(struct sk_buff *skb, struct mctp_dev *dev);

/* route output callbacks */
static int mctp_dst_discard(struct mctp_dst *dst, struct sk_buff *skb)
{
        kfree_skb(skb);
        return 0;
}

static struct mctp_sock *mctp_lookup_bind_details(struct net *net,
                                                  struct sk_buff *skb,
                                                  u8 type, u8 dest,
                                                  u8 src, bool allow_net_any)
{
        struct mctp_skb_cb *cb = mctp_cb(skb);
        struct sock *sk;
        u8 hash;

        WARN_ON_ONCE(!rcu_read_lock_held());

        hash = mctp_bind_hash(type, dest, src);

        sk_for_each_rcu(sk, &net->mctp.binds[hash]) {
                struct mctp_sock *msk = container_of(sk, struct mctp_sock, sk);

                if (!allow_net_any && msk->bind_net == MCTP_NET_ANY)
                        continue;

                if (msk->bind_net != MCTP_NET_ANY && msk->bind_net != cb->net)
                        continue;

                if (msk->bind_type != type)
                        continue;

                if (msk->bind_peer_set &&
                    !mctp_address_matches(msk->bind_peer_addr, src))
                        continue;

                if (!mctp_address_matches(msk->bind_local_addr, dest))
                        continue;

                return msk;
        }

        return NULL;
}

static struct mctp_sock *mctp_lookup_bind(struct net *net, struct sk_buff *skb)
{
        struct mctp_sock *msk;
        struct mctp_hdr *mh;
        u8 type;

        /* TODO: look up in skb->cb? */
        mh = mctp_hdr(skb);

        if (!skb_headlen(skb))
                return NULL;

        type = (*(u8 *)skb->data) & 0x7f;

        /* Look for binds in order of widening scope. A given destination or
         * source address also implies matching on a particular network.
         *
         * - Matching destination and source
         * - Matching destination
         * - Matching source
         * - Matching network, any address
         * - Any network or address
         */

        msk = mctp_lookup_bind_details(net, skb, type, mh->dest, mh->src,
                                       false);
        if (msk)
                return msk;
        msk = mctp_lookup_bind_details(net, skb, type, MCTP_ADDR_ANY, mh->src,
                                       false);
        if (msk)
                return msk;
        msk = mctp_lookup_bind_details(net, skb, type, mh->dest, MCTP_ADDR_ANY,
                                       false);
        if (msk)
                return msk;
        msk = mctp_lookup_bind_details(net, skb, type, MCTP_ADDR_ANY,
                                       MCTP_ADDR_ANY, false);
        if (msk)
                return msk;
        msk = mctp_lookup_bind_details(net, skb, type, MCTP_ADDR_ANY,
                                       MCTP_ADDR_ANY, true);
        if (msk)
                return msk;

        return NULL;
}

/* A note on the key allocations.
 *
 * struct net->mctp.keys contains our set of currently-allocated keys for
 * MCTP tag management. The lookup tuple for these is the peer EID,
 * local EID and MCTP tag.
 *
 * In some cases, the peer EID may be MCTP_EID_ANY: for example, when a
 * broadcast message is sent, we may receive responses from any peer EID.
 * Because the broadcast dest address is equivalent to ANY, we create
 * a key with (local = local-eid, peer = ANY). This allows a match on the
 * incoming broadcast responses from any peer.
 *
 * We perform lookups when packets are received, and when tags are allocated
 * in two scenarios:
 *
 *  - when a packet is sent, with a locally-owned tag: we need to find an
 *    unused tag value for the (local, peer) EID pair.
 *
 *  - when a tag is manually allocated: we need to find an unused tag value
 *    for the peer EID, but don't have a specific local EID at that stage.
 *
 * in the latter case, on successful allocation, we end up with a tag with
 * (local = ANY, peer = peer-eid).
 *
 * So, the key set allows both a local EID of ANY, as well as a peer EID of
 * ANY in the lookup tuple. Both may be ANY if we prealloc for a broadcast.
 * The matching (in mctp_key_match()) during lookup allows the match value to
 * be ANY in either the dest or source addresses.
 *
 * When allocating (+ inserting) a tag, we need to check for conflicts amongst
 * the existing tag set. This requires macthing either exactly on the local
 * and peer addresses, or either being ANY.
 */

static bool mctp_key_match(struct mctp_sk_key *key, unsigned int net,
                           mctp_eid_t local, mctp_eid_t peer, u8 tag)
{
        if (key->net != net)
                return false;

        if (!mctp_address_matches(key->local_addr, local))
                return false;

        if (!mctp_address_matches(key->peer_addr, peer))
                return false;

        if (key->tag != tag)
                return false;

        return true;
}

/* returns a key (with key->lock held, and refcounted), or NULL if no such
 * key exists.
 */
static struct mctp_sk_key *mctp_lookup_key(struct net *net, struct sk_buff *skb,
                                           unsigned int netid, mctp_eid_t peer,
                                           unsigned long *irqflags)
        __acquires(&key->lock)
{
        struct mctp_sk_key *key, *ret;
        unsigned long flags;
        struct mctp_hdr *mh;
        u8 tag;

        mh = mctp_hdr(skb);
        tag = mh->flags_seq_tag & (MCTP_HDR_TAG_MASK | MCTP_HDR_FLAG_TO);

        ret = NULL;
        spin_lock_irqsave(&net->mctp.keys_lock, flags);

        hlist_for_each_entry(key, &net->mctp.keys, hlist) {
                if (!mctp_key_match(key, netid, mh->dest, peer, tag))
                        continue;

                spin_lock(&key->lock);
                if (key->valid) {
                        refcount_inc(&key->refs);
                        ret = key;
                        break;
                }
                spin_unlock(&key->lock);
        }

        if (ret) {
                spin_unlock(&net->mctp.keys_lock);
                *irqflags = flags;
        } else {
                spin_unlock_irqrestore(&net->mctp.keys_lock, flags);
        }

        return ret;
}

static struct mctp_sk_key *mctp_key_alloc(struct mctp_sock *msk,
                                          unsigned int net,
                                          mctp_eid_t local, mctp_eid_t peer,
                                          u8 tag, gfp_t gfp)
{
        struct mctp_sk_key *key;

        key = kzalloc_obj(*key, gfp);
        if (!key)
                return NULL;

        key->net = net;
        key->peer_addr = peer;
        key->local_addr = local;
        key->tag = tag;
        key->sk = &msk->sk;
        key->valid = true;
        spin_lock_init(&key->lock);
        refcount_set(&key->refs, 1);
        sock_hold(key->sk);

        return key;
}

void mctp_key_unref(struct mctp_sk_key *key)
{
        unsigned long flags;

        if (!refcount_dec_and_test(&key->refs))
                return;

        /* even though no refs exist here, the lock allows us to stay
         * consistent with the locking requirement of mctp_dev_release_key
         */
        spin_lock_irqsave(&key->lock, flags);
        mctp_dev_release_key(key->dev, key);
        spin_unlock_irqrestore(&key->lock, flags);

        sock_put(key->sk);
        kfree(key);
}

static int mctp_key_add(struct mctp_sk_key *key, struct mctp_sock *msk)
{
        struct net *net = sock_net(&msk->sk);
        struct mctp_sk_key *tmp;
        unsigned long flags;
        int rc = 0;

        spin_lock_irqsave(&net->mctp.keys_lock, flags);

        if (sock_flag(&msk->sk, SOCK_DEAD)) {
                rc = -EINVAL;
                goto out_unlock;
        }

        hlist_for_each_entry(tmp, &net->mctp.keys, hlist) {
                if (mctp_key_match(tmp, key->net, key->local_addr,
                                   key->peer_addr, key->tag)) {
                        spin_lock(&tmp->lock);
                        if (tmp->valid)
                                rc = -EEXIST;
                        spin_unlock(&tmp->lock);
                        if (rc)
                                break;
                }
        }

        if (!rc) {
                refcount_inc(&key->refs);
                key->expiry = jiffies + mctp_key_lifetime;
                timer_reduce(&msk->key_expiry, key->expiry);

                hlist_add_head(&key->hlist, &net->mctp.keys);
                hlist_add_head(&key->sklist, &msk->keys);
        }

out_unlock:
        spin_unlock_irqrestore(&net->mctp.keys_lock, flags);

        return rc;
}

/* Helper for mctp_route_input().
 * We're done with the key; unlock and unref the key.
 * For the usual case of automatic expiry we remove the key from lists.
 * In the case that manual allocation is set on a key we release the lock
 * and local ref, reset reassembly, but don't remove from lists.
 */
static void __mctp_key_done_in(struct mctp_sk_key *key, struct net *net,
                               unsigned long flags, unsigned long reason)
__releases(&key->lock)
{
        struct sk_buff *skb;

        trace_mctp_key_release(key, reason);
        skb = key->reasm_head;
        key->reasm_head = NULL;

        if (!key->manual_alloc) {
                key->reasm_dead = true;
                key->valid = false;
                mctp_dev_release_key(key->dev, key);
        }
        spin_unlock_irqrestore(&key->lock, flags);

        if (!key->manual_alloc) {
                spin_lock_irqsave(&net->mctp.keys_lock, flags);
                if (!hlist_unhashed(&key->hlist)) {
                        hlist_del_init(&key->hlist);
                        hlist_del_init(&key->sklist);
                        mctp_key_unref(key);
                }
                spin_unlock_irqrestore(&net->mctp.keys_lock, flags);
        }

        /* and one for the local reference */
        mctp_key_unref(key);

        kfree_skb(skb);
}

#ifdef CONFIG_MCTP_FLOWS
static void mctp_skb_set_flow(struct sk_buff *skb, struct mctp_sk_key *key)
{
        struct mctp_flow *flow;

        flow = skb_ext_add(skb, SKB_EXT_MCTP);
        if (!flow)
                return;

        refcount_inc(&key->refs);
        flow->key = key;
}

static void mctp_flow_prepare_output(struct sk_buff *skb, struct mctp_dev *dev)
{
        struct mctp_sk_key *key;
        struct mctp_flow *flow;
        unsigned long flags;

        flow = skb_ext_find(skb, SKB_EXT_MCTP);
        if (!flow)
                return;

        key = flow->key;

        spin_lock_irqsave(&key->lock, flags);

        if (!key->dev)
                mctp_dev_set_key(dev, key);
        else
                WARN_ON(key->dev != dev);

        spin_unlock_irqrestore(&key->lock, flags);
}
#else
static void mctp_skb_set_flow(struct sk_buff *skb, struct mctp_sk_key *key) {}
static void mctp_flow_prepare_output(struct sk_buff *skb, struct mctp_dev *dev) {}
#endif

/* takes ownership of skb, both in success and failure cases */
static int mctp_frag_queue(struct mctp_sk_key *key, struct sk_buff *skb)
{
        struct mctp_hdr *hdr = mctp_hdr(skb);
        u8 exp_seq, this_seq;

        this_seq = (hdr->flags_seq_tag >> MCTP_HDR_SEQ_SHIFT)
                & MCTP_HDR_SEQ_MASK;

        if (!key->reasm_head) {
                /* Since we're manipulating the shared frag_list, ensure it
                 * isn't shared with any other SKBs. In the cloned case,
                 * this will free the skb; callers can no longer access it
                 * safely.
                 */
                key->reasm_head = skb_unshare(skb, GFP_ATOMIC);
                if (!key->reasm_head)
                        return -ENOMEM;

                key->reasm_tailp = &(skb_shinfo(key->reasm_head)->frag_list);
                key->last_seq = this_seq;
                return 0;
        }

        exp_seq = (key->last_seq + 1) & MCTP_HDR_SEQ_MASK;

        if (this_seq != exp_seq)
                goto err_free;

        if (key->reasm_head->len + skb->len > mctp_message_maxlen)
                goto err_free;

        skb->next = NULL;
        skb->sk = NULL;
        *key->reasm_tailp = skb;
        key->reasm_tailp = &skb->next;

        key->last_seq = this_seq;

        key->reasm_head->data_len += skb->len;
        key->reasm_head->len += skb->len;
        key->reasm_head->truesize += skb->truesize;

        return 0;

err_free:
        kfree_skb(skb);
        return -EINVAL;
}

static int mctp_dst_input(struct mctp_dst *dst, struct sk_buff *skb)
{
        struct mctp_sk_key *key, *any_key = NULL;
        struct net *net = dev_net(skb->dev);
        struct mctp_sock *msk;
        struct mctp_hdr *mh;
        unsigned int netid;
        unsigned long f;
        u8 tag, flags;
        int rc;
        u8 ver;

        msk = NULL;
        rc = -EINVAL;

        /* We may be receiving a locally-routed packet; drop source sk
         * accounting.
         *
         * From here, we will either queue the skb - either to a frag_queue, or
         * to a receiving socket. When that succeeds, we clear the skb pointer;
         * a non-NULL skb on exit will be otherwise unowned, and hence
         * kfree_skb()-ed.
         */
        skb_orphan(skb);

        if (skb->pkt_type == PACKET_OUTGOING)
                skb->pkt_type = PACKET_LOOPBACK;

        /* ensure we have enough data for a header and a type */
        if (skb->len < sizeof(struct mctp_hdr) + 1)
                goto out;

        /* grab header, advance data ptr */
        mh = mctp_hdr(skb);
        netid = mctp_cb(skb)->net;
        skb_pull(skb, sizeof(struct mctp_hdr));

        ver = mh->ver & MCTP_HDR_VER_MASK;
        if (ver < MCTP_VER_MIN || ver > MCTP_VER_MAX)
                goto out;

        flags = mh->flags_seq_tag & (MCTP_HDR_FLAG_SOM | MCTP_HDR_FLAG_EOM);
        tag = mh->flags_seq_tag & (MCTP_HDR_TAG_MASK | MCTP_HDR_FLAG_TO);

        rcu_read_lock();

        /* lookup socket / reasm context, exactly matching (src,dest,tag).
         * we hold a ref on the key, and key->lock held.
         */
        key = mctp_lookup_key(net, skb, netid, mh->src, &f);

        if (flags & MCTP_HDR_FLAG_SOM) {
                if (key) {
                        msk = container_of(key->sk, struct mctp_sock, sk);
                } else {
                        /* first response to a broadcast? do a more general
                         * key lookup to find the socket, but don't use this
                         * key for reassembly - we'll create a more specific
                         * one for future packets if required (ie, !EOM).
                         *
                         * this lookup requires key->peer to be MCTP_ADDR_ANY,
                         * it doesn't match just any key->peer.
                         */
                        any_key = mctp_lookup_key(net, skb, netid,
                                                  MCTP_ADDR_ANY, &f);
                        if (any_key) {
                                msk = container_of(any_key->sk,
                                                   struct mctp_sock, sk);
                                spin_unlock_irqrestore(&any_key->lock, f);
                        }
                }

                if (!key && !msk && (tag & MCTP_HDR_FLAG_TO))
                        msk = mctp_lookup_bind(net, skb);

                if (!msk) {
                        rc = -ENOENT;
                        goto out_unlock;
                }

                /* single-packet message? deliver to socket, clean up any
                 * pending key.
                 */
                if (flags & MCTP_HDR_FLAG_EOM) {
                        rc = sock_queue_rcv_skb(&msk->sk, skb);
                        if (!rc)
                                skb = NULL;
                        if (key) {
                                /* we've hit a pending reassembly; not much we
                                 * can do but drop it
                                 */
                                __mctp_key_done_in(key, net, f,
                                                   MCTP_TRACE_KEY_REPLIED);
                                key = NULL;
                        }
                        goto out_unlock;
                }

                /* broadcast response or a bind() - create a key for further
                 * packets for this message
                 */
                if (!key) {
                        key = mctp_key_alloc(msk, netid, mh->dest, mh->src,
                                             tag, GFP_ATOMIC);
                        if (!key) {
                                rc = -ENOMEM;
                                goto out_unlock;
                        }

                        /* we can queue without the key lock here, as the
                         * key isn't observable yet
                         */
                        mctp_frag_queue(key, skb);
                        skb = NULL;

                        /* if the key_add fails, we've raced with another
                         * SOM packet with the same src, dest and tag. There's
                         * no way to distinguish future packets, so all we
                         * can do is drop.
                         */
                        rc = mctp_key_add(key, msk);
                        if (!rc)
                                trace_mctp_key_acquire(key);

                        /* we don't need to release key->lock on exit, so
                         * clean up here and suppress the unlock via
                         * setting to NULL
                         */
                        mctp_key_unref(key);
                        key = NULL;

                } else {
                        if (key->reasm_head || key->reasm_dead) {
                                /* duplicate start? drop everything */
                                __mctp_key_done_in(key, net, f,
                                                   MCTP_TRACE_KEY_INVALIDATED);
                                rc = -EEXIST;
                                key = NULL;
                        } else {
                                rc = mctp_frag_queue(key, skb);
                                skb = NULL;
                        }
                }

        } else if (key) {
                /* this packet continues a previous message; reassemble
                 * using the message-specific key
                 */

                /* we need to be continuing an existing reassembly... */
                if (!key->reasm_head) {
                        rc = -EINVAL;
                } else {
                        rc = mctp_frag_queue(key, skb);
                        skb = NULL;
                }

                if (rc)
                        goto out_unlock;

                /* end of message? deliver to socket, and we're done with
                 * the reassembly/response key
                 */
                if (flags & MCTP_HDR_FLAG_EOM) {
                        rc = sock_queue_rcv_skb(key->sk, key->reasm_head);
                        if (!rc)
                                key->reasm_head = NULL;
                        __mctp_key_done_in(key, net, f, MCTP_TRACE_KEY_REPLIED);
                        key = NULL;
                }

        } else {
                /* not a start, no matching key */
                rc = -ENOENT;
        }

out_unlock:
        rcu_read_unlock();
        if (key) {
                spin_unlock_irqrestore(&key->lock, f);
                mctp_key_unref(key);
        }
        if (any_key)
                mctp_key_unref(any_key);
out:
        kfree_skb(skb);
        return rc;
}

static int mctp_dst_output(struct mctp_dst *dst, struct sk_buff *skb)
{
        char daddr_buf[MAX_ADDR_LEN];
        char *daddr = NULL;
        int rc;

        skb->protocol = htons(ETH_P_MCTP);
        skb->pkt_type = PACKET_OUTGOING;
        skb->dev = dst->dev->dev;

        if (skb->len > dst->mtu) {
                kfree_skb(skb);
                return -EMSGSIZE;
        }

        /* direct route; use the hwaddr we stashed in sendmsg */
        if (dst->halen) {
                if (dst->halen != skb->dev->addr_len) {
                        /* sanity check, sendmsg should have already caught this */
                        kfree_skb(skb);
                        return -EMSGSIZE;
                }
                daddr = dst->haddr;
        } else {
                /* If lookup fails let the device handle daddr==NULL */
                if (mctp_neigh_lookup(dst->dev, dst->nexthop, daddr_buf) == 0)
                        daddr = daddr_buf;
        }

        rc = dev_hard_header(skb, skb->dev, ntohs(skb->protocol),
                             daddr, skb->dev->dev_addr, skb->len);
        if (rc < 0) {
                kfree_skb(skb);
                return -EHOSTUNREACH;
        }

        mctp_flow_prepare_output(skb, dst->dev);

        rc = dev_queue_xmit(skb);
        if (rc)
                rc = net_xmit_errno(rc);

        return rc;
}

/* route alloc/release */
static void mctp_route_release(struct mctp_route *rt)
{
        if (refcount_dec_and_test(&rt->refs)) {
                if (rt->dst_type == MCTP_ROUTE_DIRECT)
                        mctp_dev_put(rt->dev);
                kfree_rcu(rt, rcu);
        }
}

/* returns a route with the refcount at 1 */
static struct mctp_route *mctp_route_alloc(void)
{
        struct mctp_route *rt;

        rt = kzalloc_obj(*rt);
        if (!rt)
                return NULL;

        INIT_LIST_HEAD(&rt->list);
        refcount_set(&rt->refs, 1);
        rt->output = mctp_dst_discard;

        return rt;
}

unsigned int mctp_default_net(struct net *net)
{
        return READ_ONCE(net->mctp.default_net);
}

int mctp_default_net_set(struct net *net, unsigned int index)
{
        if (index == 0)
                return -EINVAL;
        WRITE_ONCE(net->mctp.default_net, index);
        return 0;
}

/* tag management */
static void mctp_reserve_tag(struct net *net, struct mctp_sk_key *key,
                             struct mctp_sock *msk)
{
        struct netns_mctp *mns = &net->mctp;

        lockdep_assert_held(&mns->keys_lock);

        key->expiry = jiffies + mctp_key_lifetime;
        timer_reduce(&msk->key_expiry, key->expiry);

        /* we hold the net->key_lock here, allowing updates to both
         * then net and sk
         */
        hlist_add_head_rcu(&key->hlist, &mns->keys);
        hlist_add_head_rcu(&key->sklist, &msk->keys);
        refcount_inc(&key->refs);
}

/* Allocate a locally-owned tag value for (local, peer), and reserve
 * it for the socket msk
 */
struct mctp_sk_key *mctp_alloc_local_tag(struct mctp_sock *msk,
                                         unsigned int netid,
                                         mctp_eid_t local, mctp_eid_t peer,
                                         bool manual, u8 *tagp)
{
        struct net *net = sock_net(&msk->sk);
        struct netns_mctp *mns = &net->mctp;
        struct mctp_sk_key *key, *tmp;
        unsigned long flags;
        u8 tagbits;

        /* for NULL destination EIDs, we may get a response from any peer */
        if (peer == MCTP_ADDR_NULL)
                peer = MCTP_ADDR_ANY;

        /* be optimistic, alloc now */
        key = mctp_key_alloc(msk, netid, local, peer, 0, GFP_KERNEL);
        if (!key)
                return ERR_PTR(-ENOMEM);

        /* 8 possible tag values */
        tagbits = 0xff;

        spin_lock_irqsave(&mns->keys_lock, flags);

        /* Walk through the existing keys, looking for potential conflicting
         * tags. If we find a conflict, clear that bit from tagbits
         */
        hlist_for_each_entry(tmp, &mns->keys, hlist) {
                /* We can check the lookup fields (*_addr, tag) without the
                 * lock held, they don't change over the lifetime of the key.
                 */

                /* tags are net-specific */
                if (tmp->net != netid)
                        continue;

                /* if we don't own the tag, it can't conflict */
                if (tmp->tag & MCTP_HDR_FLAG_TO)
                        continue;

                /* Since we're avoiding conflicting entries, match peer and
                 * local addresses, including with a wildcard on ANY. See
                 * 'A note on key allocations' for background.
                 */
                if (peer != MCTP_ADDR_ANY &&
                    !mctp_address_matches(tmp->peer_addr, peer))
                        continue;

                if (local != MCTP_ADDR_ANY &&
                    !mctp_address_matches(tmp->local_addr, local))
                        continue;

                spin_lock(&tmp->lock);
                /* key must still be valid. If we find a match, clear the
                 * potential tag value
                 */
                if (tmp->valid)
                        tagbits &= ~(1 << tmp->tag);
                spin_unlock(&tmp->lock);

                if (!tagbits)
                        break;
        }

        if (tagbits) {
                key->tag = __ffs(tagbits);
                mctp_reserve_tag(net, key, msk);
                trace_mctp_key_acquire(key);

                key->manual_alloc = manual;
                *tagp = key->tag;
        }

        spin_unlock_irqrestore(&mns->keys_lock, flags);

        if (!tagbits) {
                mctp_key_unref(key);
                return ERR_PTR(-EBUSY);
        }

        return key;
}

static struct mctp_sk_key *mctp_lookup_prealloc_tag(struct mctp_sock *msk,
                                                    unsigned int netid,
                                                    mctp_eid_t daddr,
                                                    u8 req_tag, u8 *tagp)
{
        struct net *net = sock_net(&msk->sk);
        struct netns_mctp *mns = &net->mctp;
        struct mctp_sk_key *key, *tmp;
        unsigned long flags;

        req_tag &= ~(MCTP_TAG_PREALLOC | MCTP_TAG_OWNER);
        key = NULL;

        spin_lock_irqsave(&mns->keys_lock, flags);

        hlist_for_each_entry(tmp, &mns->keys, hlist) {
                if (tmp->net != netid)
                        continue;

                if (tmp->tag != req_tag)
                        continue;

                if (!mctp_address_matches(tmp->peer_addr, daddr))
                        continue;

                if (!tmp->manual_alloc)
                        continue;

                spin_lock(&tmp->lock);
                if (tmp->valid) {
                        key = tmp;
                        refcount_inc(&key->refs);
                        spin_unlock(&tmp->lock);
                        break;
                }
                spin_unlock(&tmp->lock);
        }
        spin_unlock_irqrestore(&mns->keys_lock, flags);

        if (!key)
                return ERR_PTR(-ENOENT);

        if (tagp)
                *tagp = key->tag;

        return key;
}

/* routing lookups */
static unsigned int mctp_route_netid(struct mctp_route *rt)
{
        return rt->dst_type == MCTP_ROUTE_DIRECT ?
                READ_ONCE(rt->dev->net) : rt->gateway.net;
}

static bool mctp_rt_match_eid(struct mctp_route *rt,
                              unsigned int net, mctp_eid_t eid)
{
        return mctp_route_netid(rt) == net &&
                rt->min <= eid && rt->max >= eid;
}

/* compares match, used for duplicate prevention */
static bool mctp_rt_compare_exact(struct mctp_route *rt1,
                                  struct mctp_route *rt2)
{
        ASSERT_RTNL();
        return mctp_route_netid(rt1) == mctp_route_netid(rt2) &&
                rt1->min == rt2->min &&
                rt1->max == rt2->max;
}

static mctp_eid_t mctp_dev_saddr(struct mctp_dev *dev)
{
        mctp_eid_t addr = MCTP_ADDR_NULL;
        unsigned long flags;

        spin_lock_irqsave(&dev->addrs_lock, flags);
        if (dev->num_addrs) {
                /* use the outbound interface's first address as our source */
                addr = dev->addrs[0];
        }
        spin_unlock_irqrestore(&dev->addrs_lock, flags);

        return addr;
}

/* must only be called on a direct route, as the final output hop */
static void mctp_dst_from_route(struct mctp_dst *dst, mctp_eid_t eid,
                                mctp_eid_t saddr, unsigned int mtu,
                                struct mctp_route *route)
{
        mctp_dev_hold(route->dev);
        dst->nexthop = eid;
        dst->dev = route->dev;
        dst->mtu = READ_ONCE(dst->dev->dev->mtu);
        if (mtu)
                dst->mtu = min(dst->mtu, mtu);
        dst->halen = 0;
        dst->output = route->output;
        dst->saddr = saddr;
}

int mctp_dst_from_extaddr(struct mctp_dst *dst, struct net *net, int ifindex,
                          unsigned char halen, const unsigned char *haddr)
{
        struct net_device *netdev;
        struct mctp_dev *dev;
        int rc = -ENOENT;

        if (halen > sizeof(dst->haddr))
                return -EINVAL;

        rcu_read_lock();

        netdev = dev_get_by_index_rcu(net, ifindex);
        if (!netdev)
                goto out_unlock;

        if (netdev->addr_len != halen) {
                rc = -EINVAL;
                goto out_unlock;
        }

        dev = __mctp_dev_get(netdev);
        if (!dev)
                goto out_unlock;

        dst->dev = dev;
        dst->mtu = READ_ONCE(netdev->mtu);
        dst->halen = halen;
        dst->output = mctp_dst_output;
        dst->nexthop = 0;
        dst->saddr = mctp_dev_saddr(dev);
        memcpy(dst->haddr, haddr, halen);

        rc = 0;

out_unlock:
        rcu_read_unlock();
        return rc;
}

void mctp_dst_release(struct mctp_dst *dst)
{
        mctp_dev_put(dst->dev);
}

static struct mctp_route *mctp_route_lookup_single(struct net *net,
                                                   unsigned int dnet,
                                                   mctp_eid_t daddr)
{
        struct mctp_route *rt;

        list_for_each_entry_rcu(rt, &net->mctp.routes, list) {
                if (mctp_rt_match_eid(rt, dnet, daddr))
                        return rt;
        }

        return NULL;
}

/* populates *dst on successful lookup, if set */
int mctp_route_lookup(struct net *net, unsigned int dnet,
                      mctp_eid_t daddr, struct mctp_dst *dst)
{
        const unsigned int max_depth = 32;
        unsigned int depth, mtu = 0;
        int rc = -EHOSTUNREACH;

        rcu_read_lock();

        for (depth = 0; depth < max_depth; depth++) {
                struct mctp_route *rt;

                rt = mctp_route_lookup_single(net, dnet, daddr);
                if (!rt)
                        break;

                /* clamp mtu to the smallest in the path, allowing 0
                 * to specify no restrictions
                 */
                if (mtu && rt->mtu)
                        mtu = min(mtu, rt->mtu);
                else
                        mtu = mtu ?: rt->mtu;

                if (rt->dst_type == MCTP_ROUTE_DIRECT) {
                        mctp_eid_t saddr;

                        /* rt->dev may be going away concurrently: its last
                         * reference is dropped in mctp_dev_put(), which frees
                         * mdev->addrs that mctp_dev_saddr() reads, and
                         * mctp_dst_from_route() takes a reference on it.  Pin
                         * it before use, and skip a device that is already
                         * dead rather than resurrecting it.
                         */
                        if (!refcount_inc_not_zero(&rt->dev->refs))
                                break;

                        saddr = mctp_dev_saddr(rt->dev);

                        /* cannot do gateway-ed routes without a src  */
                        if (saddr == MCTP_ADDR_NULL && depth != 0) {
                                mctp_dev_put(rt->dev);
                                break;
                        }

                        if (dst)
                                mctp_dst_from_route(dst, daddr, saddr, mtu, rt);
                        mctp_dev_put(rt->dev);
                        rc = 0;
                        break;

                } else if (rt->dst_type == MCTP_ROUTE_GATEWAY) {
                        daddr = rt->gateway.eid;
                }
        }

        rcu_read_unlock();

        return rc;
}

static int mctp_dst_input_null(struct net *net, struct net_device *dev,
                               struct mctp_dst *dst)
{
        rcu_read_lock();
        dst->dev = __mctp_dev_get(dev);
        rcu_read_unlock();

        if (!dst->dev)
                return -EHOSTUNREACH;

        dst->mtu = READ_ONCE(dev->mtu);
        dst->halen = 0;
        dst->output = mctp_dst_input;
        dst->nexthop = 0;

        return 0;
}

static int mctp_do_fragment_route(struct mctp_dst *dst, struct sk_buff *skb,
                                  unsigned int mtu, u8 tag)
{
        const unsigned int hlen = sizeof(struct mctp_hdr);
        struct mctp_hdr *hdr, *hdr2;
        unsigned int pos, size, headroom;
        struct sk_buff *skb2;
        int rc;
        u8 seq;

        hdr = mctp_hdr(skb);
        seq = 0;
        rc = 0;

        if (mtu < hlen + 1) {
                kfree_skb(skb);
                return -EMSGSIZE;
        }

        /* within MTU? avoid the copy, send original skb */
        if (skb->len <= mtu) {
                hdr->flags_seq_tag = MCTP_HDR_FLAG_SOM |
                        MCTP_HDR_FLAG_EOM | tag;
                return dst->output(dst, skb);
        }

        /* keep same headroom as the original skb */
        headroom = skb_headroom(skb);

        /* we've got the header */
        skb_pull(skb, hlen);

        for (pos = 0; pos < skb->len;) {
                /* size of message payload */
                size = min(mtu - hlen, skb->len - pos);

                skb2 = alloc_skb(headroom + hlen + size, GFP_KERNEL);
                if (!skb2) {
                        rc = -ENOMEM;
                        break;
                }

                /* generic skb copy */
                skb2->protocol = skb->protocol;
                skb2->priority = skb->priority;
                skb2->dev = skb->dev;
                memcpy(skb2->cb, skb->cb, sizeof(skb2->cb));

                if (skb->sk)
                        skb_set_owner_w(skb2, skb->sk);

                /* establish packet */
                skb_reserve(skb2, headroom);
                skb_reset_network_header(skb2);
                skb_put(skb2, hlen + size);
                skb2->transport_header = skb2->network_header + hlen;

                /* copy header fields, calculate SOM/EOM flags & seq */
                hdr2 = mctp_hdr(skb2);
                hdr2->ver = hdr->ver;
                hdr2->dest = hdr->dest;
                hdr2->src = hdr->src;
                hdr2->flags_seq_tag = tag &
                        (MCTP_HDR_TAG_MASK | MCTP_HDR_FLAG_TO);

                if (pos == 0)
                        hdr2->flags_seq_tag |= MCTP_HDR_FLAG_SOM;

                if (pos + size == skb->len)
                        hdr2->flags_seq_tag |= MCTP_HDR_FLAG_EOM;

                hdr2->flags_seq_tag |= seq << MCTP_HDR_SEQ_SHIFT;

                /* copy message payload */
                skb_copy_bits(skb, pos, skb_transport_header(skb2), size);

                /* we need to copy the extensions, for MCTP flow data */
                skb_ext_copy(skb2, skb);

                /* do route */
                rc = dst->output(dst, skb2);
                if (rc)
                        break;

                seq = (seq + 1) & MCTP_HDR_SEQ_MASK;
                pos += size;
        }

        consume_skb(skb);
        return rc;
}

int mctp_local_output(struct sock *sk, struct mctp_dst *dst,
                      struct sk_buff *skb, mctp_eid_t daddr, u8 req_tag)
{
        struct mctp_sock *msk = container_of(sk, struct mctp_sock, sk);
        struct mctp_sk_key *key;
        struct mctp_hdr *hdr;
        unsigned int netid;
        u8 tag;

        KUNIT_STATIC_STUB_REDIRECT(mctp_local_output, sk, dst, skb, daddr,
                                   req_tag);

        netid = READ_ONCE(dst->dev->net);

        if (req_tag & MCTP_TAG_OWNER) {
                if (req_tag & MCTP_TAG_PREALLOC)
                        key = mctp_lookup_prealloc_tag(msk, netid, daddr,
                                                       req_tag, &tag);
                else
                        key = mctp_alloc_local_tag(msk, netid, dst->saddr,
                                                   daddr, false, &tag);

                if (IS_ERR(key)) {
                        kfree_skb(skb);
                        return PTR_ERR(key);
                }
                mctp_skb_set_flow(skb, key);
                /* done with the key in this scope */
                mctp_key_unref(key);
                tag |= MCTP_HDR_FLAG_TO;
        } else {
                key = NULL;
                tag = req_tag & MCTP_TAG_MASK;
        }

        skb->pkt_type = PACKET_OUTGOING;
        skb->protocol = htons(ETH_P_MCTP);
        skb->priority = 0;
        skb_reset_transport_header(skb);
        skb_push(skb, sizeof(struct mctp_hdr));
        skb_reset_network_header(skb);
        skb->dev = dst->dev->dev;

        /* set up common header fields */
        hdr = mctp_hdr(skb);
        hdr->ver = 1;
        hdr->dest = daddr;
        hdr->src = dst->saddr;

        /* route output functions consume the skb, even on error */
        return mctp_do_fragment_route(dst, skb, dst->mtu, tag);
}

/* route management */

/* mctp_route_add(): Add the provided route, previously allocated via
 * mctp_route_alloc(). On success, takes ownership of @rt, which includes a
 * hold on rt->dev for usage in the route table. On failure a caller will want
 * to mctp_route_release().
 *
 * We expect that the caller has set rt->type, rt->dst_type, rt->min, rt->max,
 * rt->mtu and either rt->dev (with a reference held appropriately) or
 * rt->gateway. Other fields will be populated.
 */
static int mctp_route_add(struct net *net, struct mctp_route *rt)
{
        struct mctp_route *ert;

        if (!mctp_address_unicast(rt->min) || !mctp_address_unicast(rt->max))
                return -EINVAL;

        if (rt->dst_type == MCTP_ROUTE_DIRECT && !rt->dev)
                return -EINVAL;

        if (rt->dst_type == MCTP_ROUTE_GATEWAY && !rt->gateway.eid)
                return -EINVAL;

        switch (rt->type) {
        case RTN_LOCAL:
                rt->output = mctp_dst_input;
                break;
        case RTN_UNICAST:
                rt->output = mctp_dst_output;
                break;
        default:
                return -EINVAL;
        }

        ASSERT_RTNL();

        /* Prevent duplicate identical routes. */
        list_for_each_entry(ert, &net->mctp.routes, list) {
                if (mctp_rt_compare_exact(rt, ert)) {
                        return -EEXIST;
                }
        }

        list_add_rcu(&rt->list, &net->mctp.routes);

        return 0;
}

static int mctp_route_remove(struct net *net, unsigned int netid,
                             mctp_eid_t daddr_start, unsigned int daddr_extent,
                             unsigned char type)
{
        struct mctp_route *rt, *tmp;
        mctp_eid_t daddr_end;
        bool dropped;

        if (daddr_extent > 0xff || daddr_start + daddr_extent >= 255)
                return -EINVAL;

        daddr_end = daddr_start + daddr_extent;
        dropped = false;

        ASSERT_RTNL();

        list_for_each_entry_safe(rt, tmp, &net->mctp.routes, list) {
                if (mctp_route_netid(rt) == netid &&
                    rt->min == daddr_start && rt->max == daddr_end &&
                    rt->type == type) {
                        list_del_rcu(&rt->list);
                        /* TODO: immediate RTM_DELROUTE */
                        mctp_route_release(rt);
                        dropped = true;
                }
        }

        return dropped ? 0 : -ENOENT;
}

int mctp_route_add_local(struct mctp_dev *mdev, mctp_eid_t addr)
{
        struct mctp_route *rt;
        int rc;

        rt = mctp_route_alloc();
        if (!rt)
                return -ENOMEM;

        rt->min = addr;
        rt->max = addr;
        rt->dst_type = MCTP_ROUTE_DIRECT;
        rt->dev = mdev;
        rt->type = RTN_LOCAL;

        mctp_dev_hold(rt->dev);

        rc = mctp_route_add(dev_net(mdev->dev), rt);
        if (rc)
                mctp_route_release(rt);

        return rc;
}

int mctp_route_remove_local(struct mctp_dev *mdev, mctp_eid_t addr)
{
        return mctp_route_remove(dev_net(mdev->dev), mdev->net,
                                 addr, 0, RTN_LOCAL);
}

/* removes all entries for a given device */
void mctp_route_remove_dev(struct mctp_dev *mdev)
{
        struct net *net = dev_net(mdev->dev);
        struct mctp_route *rt, *tmp;

        ASSERT_RTNL();
        list_for_each_entry_safe(rt, tmp, &net->mctp.routes, list) {
                if (rt->dst_type == MCTP_ROUTE_DIRECT && rt->dev == mdev) {
                        list_del_rcu(&rt->list);
                        /* TODO: immediate RTM_DELROUTE */
                        mctp_route_release(rt);
                }
        }
}

/* Incoming packet-handling */

static int mctp_pkttype_receive(struct sk_buff *skb, struct net_device *dev,
                                struct packet_type *pt,
                                struct net_device *orig_dev)
{
        struct net *net = dev_net(dev);
        struct mctp_dev *mdev;
        struct mctp_skb_cb *cb;
        struct mctp_dst dst;
        struct mctp_hdr *mh;
        int rc;
        u8 ver;

        rcu_read_lock();
        mdev = __mctp_dev_get(dev);
        rcu_read_unlock();
        if (!mdev) {
                /* basic non-data sanity checks */
                goto err_drop;
        }

        if (!pskb_may_pull(skb, sizeof(struct mctp_hdr)))
                goto err_drop;

        skb_reset_transport_header(skb);
        skb_reset_network_header(skb);

        /* We have enough for a header; decode and route */
        mh = mctp_hdr(skb);
        ver = mh->ver & MCTP_HDR_VER_MASK;
        if (ver < MCTP_VER_MIN || ver > MCTP_VER_MAX)
                goto err_drop;

        /* source must be valid unicast or null; drop reserved ranges and
         * broadcast
         */
        if (!(mctp_address_unicast(mh->src) || mctp_address_null(mh->src)))
                goto err_drop;

        /* dest address: as above, but allow broadcast */
        if (!(mctp_address_unicast(mh->dest) || mctp_address_null(mh->dest) ||
              mctp_address_broadcast(mh->dest)))
                goto err_drop;

        /* MCTP drivers must populate halen/haddr */
        if (dev->type == ARPHRD_MCTP) {
                cb = mctp_cb(skb);
        } else {
                cb = __mctp_cb(skb);
                cb->halen = 0;
        }
        cb->net = READ_ONCE(mdev->net);
        cb->ifindex = dev->ifindex;

        rc = mctp_route_lookup(net, cb->net, mh->dest, &dst);

        /* NULL EID, but addressed to our physical address */
        if (rc && mh->dest == MCTP_ADDR_NULL && skb->pkt_type == PACKET_HOST)
                rc = mctp_dst_input_null(net, dev, &dst);

        if (rc)
                goto err_drop;

        dst.output(&dst, skb);
        mctp_dst_release(&dst);
        mctp_dev_put(mdev);

        return NET_RX_SUCCESS;

err_drop:
        kfree_skb(skb);
        mctp_dev_put(mdev);
        return NET_RX_DROP;
}

static struct packet_type mctp_packet_type = {
        .type = cpu_to_be16(ETH_P_MCTP),
        .func = mctp_pkttype_receive,
};

/* netlink interface */

static const struct nla_policy rta_mctp_policy[RTA_MAX + 1] = {
        [RTA_DST]                = { .type = NLA_U8 },
        [RTA_METRICS]                = { .type = NLA_NESTED },
        [RTA_OIF]                = { .type = NLA_U32 },
        [RTA_GATEWAY]                = NLA_POLICY_EXACT_LEN(sizeof(struct mctp_fq_addr)),
};

static const struct nla_policy rta_metrics_policy[RTAX_MAX + 1] = {
        [RTAX_MTU]                = { .type = NLA_U32 },
};

/* base parsing; common to both _lookup and _populate variants.
 *
 * For gateway routes (which have a RTA_GATEWAY, and no RTA_OIF), we populate
 * *gatweayp. for direct routes (RTA_OIF, no RTA_GATEWAY), we populate *mdev.
 */
static int mctp_route_nlparse_common(struct net *net, struct nlmsghdr *nlh,
                                     struct netlink_ext_ack *extack,
                                     struct nlattr **tb, struct rtmsg **rtm,
                                     struct mctp_dev **mdev,
                                     struct mctp_fq_addr *gatewayp,
                                     mctp_eid_t *daddr_start)
{
        struct mctp_fq_addr *gateway = NULL;
        unsigned int ifindex = 0;
        struct net_device *dev;
        int rc;

        rc = nlmsg_parse(nlh, sizeof(struct rtmsg), tb, RTA_MAX,
                         rta_mctp_policy, extack);
        if (rc < 0) {
                NL_SET_ERR_MSG(extack, "incorrect format");
                return rc;
        }

        if (!tb[RTA_DST]) {
                NL_SET_ERR_MSG(extack, "dst EID missing");
                return -EINVAL;
        }
        *daddr_start = nla_get_u8(tb[RTA_DST]);

        if (tb[RTA_OIF])
                ifindex = nla_get_u32(tb[RTA_OIF]);

        if (tb[RTA_GATEWAY])
                gateway = nla_data(tb[RTA_GATEWAY]);

        if (ifindex && gateway) {
                NL_SET_ERR_MSG(extack,
                               "cannot specify both ifindex and gateway");
                return -EINVAL;

        } else if (ifindex) {
                dev = __dev_get_by_index(net, ifindex);
                if (!dev) {
                        NL_SET_ERR_MSG(extack, "bad ifindex");
                        return -ENODEV;
                }
                *mdev = mctp_dev_get_rtnl(dev);
                if (!*mdev)
                        return -ENODEV;
                gatewayp->eid = 0;

        } else if (gateway) {
                if (!mctp_address_unicast(gateway->eid)) {
                        NL_SET_ERR_MSG(extack, "bad gateway");
                        return -EINVAL;
                }

                gatewayp->eid = gateway->eid;
                gatewayp->net = gateway->net != MCTP_NET_ANY ?
                        gateway->net :
                        READ_ONCE(net->mctp.default_net);
                *mdev = NULL;

        } else {
                NL_SET_ERR_MSG(extack, "no route output provided");
                return -EINVAL;
        }

        *rtm = nlmsg_data(nlh);
        if ((*rtm)->rtm_family != AF_MCTP) {
                NL_SET_ERR_MSG(extack, "route family must be AF_MCTP");
                return -EINVAL;
        }

        if ((*rtm)->rtm_type != RTN_UNICAST) {
                NL_SET_ERR_MSG(extack, "rtm_type must be RTN_UNICAST");
                return -EINVAL;
        }

        return 0;
}

/* Route parsing for lookup operations; we only need the "route target"
 * components (ie., network and dest-EID range).
 */
static int mctp_route_nlparse_lookup(struct net *net, struct nlmsghdr *nlh,
                                     struct netlink_ext_ack *extack,
                                     unsigned char *type, unsigned int *netid,
                                     mctp_eid_t *daddr_start,
                                     unsigned int *daddr_extent)
{
        struct nlattr *tb[RTA_MAX + 1];
        struct mctp_fq_addr gw;
        struct mctp_dev *mdev;
        struct rtmsg *rtm;
        int rc;

        rc = mctp_route_nlparse_common(net, nlh, extack, tb, &rtm,
                                       &mdev, &gw, daddr_start);
        if (rc)
                return rc;

        if (mdev) {
                *netid = mdev->net;
        } else if (gw.eid) {
                *netid = gw.net;
        } else {
                /* bug: _nlparse_common should not allow this */
                return -1;
        }

        *type = rtm->rtm_type;
        *daddr_extent = rtm->rtm_dst_len;

        return 0;
}

/* Full route parse for RTM_NEWROUTE: populate @rt. On success,
 * MCTP_ROUTE_DIRECT routes (ie, those with a direct dev) will hold a reference
 * to that dev.
 */
static int mctp_route_nlparse_populate(struct net *net, struct nlmsghdr *nlh,
                                       struct netlink_ext_ack *extack,
                                       struct mctp_route *rt)
{
        struct nlattr *tbx[RTAX_MAX + 1];
        struct nlattr *tb[RTA_MAX + 1];
        unsigned int daddr_extent;
        struct mctp_fq_addr gw;
        mctp_eid_t daddr_start;
        struct mctp_dev *dev;
        struct rtmsg *rtm;
        u32 mtu = 0;
        int rc;

        rc = mctp_route_nlparse_common(net, nlh, extack, tb, &rtm,
                                       &dev, &gw, &daddr_start);
        if (rc)
                return rc;

        daddr_extent = rtm->rtm_dst_len;

        if (daddr_extent > 0xff || daddr_extent + daddr_start >= 255) {
                NL_SET_ERR_MSG(extack, "invalid eid range");
                return -EINVAL;
        }

        if (tb[RTA_METRICS]) {
                rc = nla_parse_nested(tbx, RTAX_MAX, tb[RTA_METRICS],
                                      rta_metrics_policy, NULL);
                if (rc < 0) {
                        NL_SET_ERR_MSG(extack, "incorrect RTA_METRICS format");
                        return rc;
                }
                if (tbx[RTAX_MTU])
                        mtu = nla_get_u32(tbx[RTAX_MTU]);
        }

        rt->type = rtm->rtm_type;
        rt->min = daddr_start;
        rt->max = daddr_start + daddr_extent;
        rt->mtu = mtu;
        if (gw.eid) {
                rt->dst_type = MCTP_ROUTE_GATEWAY;
                rt->gateway.eid = gw.eid;
                rt->gateway.net = gw.net;
        } else {
                rt->dst_type = MCTP_ROUTE_DIRECT;
                rt->dev = dev;
                mctp_dev_hold(rt->dev);
        }

        return 0;
}

static int mctp_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
                         struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct mctp_route *rt;
        int rc;

        rt = mctp_route_alloc();
        if (!rt)
                return -ENOMEM;

        rc = mctp_route_nlparse_populate(net, nlh, extack, rt);
        if (rc < 0)
                goto err_free;

        if (rt->dst_type == MCTP_ROUTE_DIRECT &&
            rt->dev->dev->flags & IFF_LOOPBACK) {
                NL_SET_ERR_MSG(extack, "no routes to loopback");
                rc = -EINVAL;
                goto err_free;
        }

        rc = mctp_route_add(net, rt);
        if (!rc)
                return 0;

err_free:
        mctp_route_release(rt);
        return rc;
}

static int mctp_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
                         struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        unsigned int netid, daddr_extent;
        unsigned char type = RTN_UNSPEC;
        mctp_eid_t daddr_start;
        int rc;

        rc = mctp_route_nlparse_lookup(net, nlh, extack, &type, &netid,
                                       &daddr_start, &daddr_extent);
        if (rc < 0)
                return rc;

        /* we only have unicast routes */
        if (type != RTN_UNICAST)
                return -EINVAL;

        rc = mctp_route_remove(net, netid, daddr_start, daddr_extent, type);
        return rc;
}

static int mctp_fill_rtinfo(struct sk_buff *skb, struct mctp_route *rt,
                            u32 portid, u32 seq, int event, unsigned int flags)
{
        struct nlmsghdr *nlh;
        struct rtmsg *hdr;
        void *metrics;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(*hdr), flags);
        if (!nlh)
                return -EMSGSIZE;

        hdr = nlmsg_data(nlh);
        memset(hdr, 0, sizeof(*hdr));
        hdr->rtm_family = AF_MCTP;

        /* we use the _len fields as a number of EIDs, rather than
         * a number of bits in the address
         */
        hdr->rtm_dst_len = rt->max - rt->min;
        hdr->rtm_src_len = 0;
        hdr->rtm_tos = 0;
        hdr->rtm_table = RT_TABLE_DEFAULT;
        hdr->rtm_protocol = RTPROT_STATIC; /* everything is user-defined */
        hdr->rtm_type = rt->type;

        if (nla_put_u8(skb, RTA_DST, rt->min))
                goto cancel;

        metrics = nla_nest_start_noflag(skb, RTA_METRICS);
        if (!metrics)
                goto cancel;

        if (rt->mtu) {
                if (nla_put_u32(skb, RTAX_MTU, rt->mtu))
                        goto cancel;
        }

        nla_nest_end(skb, metrics);

        if (rt->dst_type == MCTP_ROUTE_DIRECT) {
                hdr->rtm_scope = RT_SCOPE_LINK;
                if (nla_put_u32(skb, RTA_OIF, rt->dev->dev->ifindex))
                        goto cancel;
        } else if (rt->dst_type == MCTP_ROUTE_GATEWAY) {
                hdr->rtm_scope = RT_SCOPE_UNIVERSE;
                if (nla_put(skb, RTA_GATEWAY,
                            sizeof(rt->gateway), &rt->gateway))
                        goto cancel;
        }

        nlmsg_end(skb, nlh);

        return 0;

cancel:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int mctp_dump_rtinfo(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct net *net = sock_net(skb->sk);
        struct mctp_route *rt;
        int s_idx, idx;

        /* TODO: allow filtering on route data, possibly under
         * cb->strict_check
         */

        /* TODO: change to struct overlay */
        s_idx = cb->args[0];
        idx = 0;

        rcu_read_lock();
        list_for_each_entry_rcu(rt, &net->mctp.routes, list) {
                if (idx++ < s_idx)
                        continue;
                if (mctp_fill_rtinfo(skb, rt,
                                     NETLINK_CB(cb->skb).portid,
                                     cb->nlh->nlmsg_seq,
                                     RTM_NEWROUTE, NLM_F_MULTI) < 0)
                        break;
        }

        rcu_read_unlock();
        cb->args[0] = idx;

        return skb->len;
}

/* net namespace implementation */
static int __net_init mctp_routes_net_init(struct net *net)
{
        struct netns_mctp *ns = &net->mctp;

        INIT_LIST_HEAD(&ns->routes);
        hash_init(ns->binds);
        mutex_init(&ns->bind_lock);
        INIT_HLIST_HEAD(&ns->keys);
        spin_lock_init(&ns->keys_lock);
        WARN_ON(mctp_default_net_set(net, MCTP_INITIAL_DEFAULT_NET));
        return 0;
}

static void __net_exit mctp_routes_net_exit(struct net *net)
{
        struct mctp_route *rt;

        rcu_read_lock();
        list_for_each_entry_rcu(rt, &net->mctp.routes, list)
                mctp_route_release(rt);
        rcu_read_unlock();
}

static struct pernet_operations mctp_net_ops = {
        .init = mctp_routes_net_init,
        .exit = mctp_routes_net_exit,
};

static const struct rtnl_msg_handler mctp_route_rtnl_msg_handlers[] = {
        {THIS_MODULE, PF_MCTP, RTM_NEWROUTE, mctp_newroute, NULL, 0},
        {THIS_MODULE, PF_MCTP, RTM_DELROUTE, mctp_delroute, NULL, 0},
        {THIS_MODULE, PF_MCTP, RTM_GETROUTE, NULL, mctp_dump_rtinfo, 0},
};

int __init mctp_routes_init(void)
{
        int err;

        dev_add_pack(&mctp_packet_type);

        err = register_pernet_subsys(&mctp_net_ops);
        if (err)
                goto err_pernet;

        err = rtnl_register_many(mctp_route_rtnl_msg_handlers);
        if (err)
                goto err_rtnl;

        return 0;

err_rtnl:
        unregister_pernet_subsys(&mctp_net_ops);
err_pernet:
        dev_remove_pack(&mctp_packet_type);
        return err;
}

void mctp_routes_exit(void)
{
        rtnl_unregister_many(mctp_route_rtnl_msg_handlers);
        unregister_pernet_subsys(&mctp_net_ops);
        dev_remove_pack(&mctp_packet_type);
}

#if IS_ENABLED(CONFIG_MCTP_TEST)
#include "test/route-test.c"
#endif





































   64 





   64 














  284 











   37 
















   37 









   37 







   37 











   36 







    1 
    4 







    4 





























    4 







































































    4 


    1 








   16 









   16 



































    4 
























   34 




   33 


    4 






    4 






























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
// SPDX-License-Identifier: GPL-2.0-or-later
/* SCTP kernel implementation
 * (C) Copyright IBM Corp. 2001, 2004
 * Copyright (c) 1999-2000 Cisco, Inc.
 * Copyright (c) 1999-2001 Motorola, Inc.
 * Copyright (c) 2001 Intel Corp.
 *
 * This file is part of the SCTP kernel implementation
 *
 * These functions manipulate sctp tsn mapping array.
 *
 * Please send any bug reports or fixes you make to the
 * email address(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *    La Monte H.P. Yarroll <piggy@acm.org>
 *    Jon Grimm             <jgrimm@us.ibm.com>
 *    Karl Knutson          <karl@athena.chicago.il.us>
 *    Sridhar Samudrala     <sri@us.ibm.com>
 */

#include <linux/slab.h>
#include <linux/types.h>
#include <linux/bitmap.h>
#include <net/sctp/sctp.h>
#include <net/sctp/sm.h>

static void sctp_tsnmap_update(struct sctp_tsnmap *map);
static void sctp_tsnmap_find_gap_ack(unsigned long *map, __u16 off,
                                     __u16 len, __u16 *start, __u16 *end);
static int sctp_tsnmap_grow(struct sctp_tsnmap *map, u16 size);

/* Initialize a block of memory as a tsnmap.  */
struct sctp_tsnmap *sctp_tsnmap_init(struct sctp_tsnmap *map, __u16 len,
                                     __u32 initial_tsn, gfp_t gfp)
{
        if (!map->tsn_map) {
                map->tsn_map = kzalloc(len>>3, gfp);
                if (map->tsn_map == NULL)
                        return NULL;

                map->len = len;
        } else {
                bitmap_zero(map->tsn_map, map->len);
        }

        /* Keep track of TSNs represented by tsn_map.  */
        map->base_tsn = initial_tsn;
        map->cumulative_tsn_ack_point = initial_tsn - 1;
        map->max_tsn_seen = map->cumulative_tsn_ack_point;
        map->num_dup_tsns = 0;

        return map;
}

void sctp_tsnmap_free(struct sctp_tsnmap *map)
{
        map->len = 0;
        kfree(map->tsn_map);
}

/* Test the tracking state of this TSN.
 * Returns:
 *   0 if the TSN has not yet been seen
 *  >0 if the TSN has been seen (duplicate)
 *  <0 if the TSN is invalid (too large to track)
 */
int sctp_tsnmap_check(const struct sctp_tsnmap *map, __u32 tsn)
{
        u32 gap;

        /* Check to see if this is an old TSN */
        if (TSN_lte(tsn, map->cumulative_tsn_ack_point))
                return 1;

        /* Verify that we can hold this TSN and that it will not
         * overflow our map
         */
        if (!TSN_lt(tsn, map->base_tsn + SCTP_TSN_MAP_SIZE))
                return -1;

        /* Calculate the index into the mapping arrays.  */
        gap = tsn - map->base_tsn;

        /* Check to see if TSN has already been recorded.  */
        if (gap < map->len && test_bit(gap, map->tsn_map))
                return 1;
        else
                return 0;
}


/* Mark this TSN as seen.  */
int sctp_tsnmap_mark(struct sctp_tsnmap *map, __u32 tsn,
                     struct sctp_transport *trans)
{
        u16 gap;

        if (TSN_lt(tsn, map->base_tsn))
                return 0;

        gap = tsn - map->base_tsn;

        if (gap >= map->len && !sctp_tsnmap_grow(map, gap + 1))
                return -ENOMEM;

        if (!sctp_tsnmap_has_gap(map) && gap == 0) {
                /* In this case the map has no gaps and the tsn we are
                 * recording is the next expected tsn.  We don't touch
                 * the map but simply bump the values.
                 */
                map->max_tsn_seen++;
                map->cumulative_tsn_ack_point++;
                if (trans)
                        trans->sack_generation =
                                trans->asoc->peer.sack_generation;
                map->base_tsn++;
        } else {
                /* Either we already have a gap, or about to record a gap, so
                 * have work to do.
                 *
                 * Bump the max.
                 */
                if (TSN_lt(map->max_tsn_seen, tsn))
                        map->max_tsn_seen = tsn;

                /* Mark the TSN as received.  */
                set_bit(gap, map->tsn_map);

                /* Go fixup any internal TSN mapping variables including
                 * cumulative_tsn_ack_point.
                 */
                sctp_tsnmap_update(map);
        }

        return 0;
}


/* Initialize a Gap Ack Block iterator from memory being provided.  */
static void sctp_tsnmap_iter_init(const struct sctp_tsnmap *map,
                                  struct sctp_tsnmap_iter *iter)
{
        /* Only start looking one past the Cumulative TSN Ack Point.  */
        iter->start = map->cumulative_tsn_ack_point + 1;
}

/* Get the next Gap Ack Blocks. Returns 0 if there was not another block
 * to get.
 */
static int sctp_tsnmap_next_gap_ack(const struct sctp_tsnmap *map,
                                    struct sctp_tsnmap_iter *iter,
                                    __u16 *start, __u16 *end)
{
        int ended = 0;
        __u16 start_ = 0, end_ = 0, offset;

        /* If there are no more gap acks possible, get out fast.  */
        if (TSN_lte(map->max_tsn_seen, iter->start))
                return 0;

        offset = iter->start - map->base_tsn;
        sctp_tsnmap_find_gap_ack(map->tsn_map, offset, map->len,
                                 &start_, &end_);

        /* The Gap Ack Block happens to end at the end of the map. */
        if (start_ && !end_)
                end_ = map->len - 1;

        /* If we found a Gap Ack Block, return the start and end and
         * bump the iterator forward.
         */
        if (end_) {
                /* Fix up the start and end based on the
                 * Cumulative TSN Ack which is always 1 behind base.
                 */
                *start = start_ + 1;
                *end = end_ + 1;

                /* Move the iterator forward.  */
                iter->start = map->cumulative_tsn_ack_point + *end + 1;
                ended = 1;
        }

        return ended;
}

/* Mark this and any lower TSN as seen.  */
void sctp_tsnmap_skip(struct sctp_tsnmap *map, __u32 tsn)
{
        u32 gap;

        if (TSN_lt(tsn, map->base_tsn))
                return;
        if (!TSN_lt(tsn, map->base_tsn + SCTP_TSN_MAP_SIZE))
                return;

        /* Bump the max.  */
        if (TSN_lt(map->max_tsn_seen, tsn))
                map->max_tsn_seen = tsn;

        gap = tsn - map->base_tsn + 1;

        map->base_tsn += gap;
        map->cumulative_tsn_ack_point += gap;
        if (gap >= map->len) {
                /* If our gap is larger then the map size, just
                 * zero out the map.
                 */
                bitmap_zero(map->tsn_map, map->len);
        } else {
                /* If the gap is smaller than the map size,
                 * shift the map by 'gap' bits and update further.
                 */
                bitmap_shift_right(map->tsn_map, map->tsn_map, gap, map->len);
                sctp_tsnmap_update(map);
        }
}

/********************************************************************
 * 2nd Level Abstractions
 ********************************************************************/

/* This private helper function updates the tsnmap buffers and
 * the Cumulative TSN Ack Point.
 */
static void sctp_tsnmap_update(struct sctp_tsnmap *map)
{
        u16 len;
        unsigned long zero_bit;


        len = map->max_tsn_seen - map->cumulative_tsn_ack_point;
        zero_bit = find_first_zero_bit(map->tsn_map, len);
        if (!zero_bit)
                return;                /* The first 0-bit is bit 0.  nothing to do */

        map->base_tsn += zero_bit;
        map->cumulative_tsn_ack_point += zero_bit;

        bitmap_shift_right(map->tsn_map, map->tsn_map, zero_bit, map->len);
}

/* How many data chunks  are we missing from our peer?
 */
__u16 sctp_tsnmap_pending(struct sctp_tsnmap *map)
{
        __u32 cum_tsn = map->cumulative_tsn_ack_point;
        __u32 max_tsn = map->max_tsn_seen;
        __u32 base_tsn = map->base_tsn;
        __u16 pending_data;
        u32 gap;

        pending_data = max_tsn - cum_tsn;
        gap = max_tsn - base_tsn;

        if (gap == 0 || gap >= map->len)
                goto out;

        pending_data -= bitmap_weight(map->tsn_map, gap + 1);
out:
        return pending_data;
}

/* This is a private helper for finding Gap Ack Blocks.  It searches a
 * single array for the start and end of a Gap Ack Block.
 *
 * The flags "started" and "ended" tell is if we found the beginning
 * or (respectively) the end of a Gap Ack Block.
 */
static void sctp_tsnmap_find_gap_ack(unsigned long *map, __u16 off,
                                     __u16 len, __u16 *start, __u16 *end)
{
        int i = off;

        /* Look through the entire array, but break out
         * early if we have found the end of the Gap Ack Block.
         */

        /* Also, stop looking past the maximum TSN seen. */

        /* Look for the start. */
        i = find_next_bit(map, len, off);
        if (i < len)
                *start = i;

        /* Look for the end.  */
        if (*start) {
                /* We have found the start, let's find the
                 * end.  If we find the end, break out.
                 */
                i = find_next_zero_bit(map, len, i);
                if (i < len)
                        *end = i - 1;
        }
}

/* Renege that we have seen a TSN.  */
void sctp_tsnmap_renege(struct sctp_tsnmap *map, __u32 tsn)
{
        u32 gap;

        if (TSN_lt(tsn, map->base_tsn))
                return;
        /* Assert: TSN is in range.  */
        if (!TSN_lt(tsn, map->base_tsn + map->len))
                return;

        gap = tsn - map->base_tsn;

        /* Pretend we never saw the TSN.  */
        clear_bit(gap, map->tsn_map);
}

/* How many gap ack blocks do we have recorded? */
__u16 sctp_tsnmap_num_gabs(struct sctp_tsnmap *map,
                           struct sctp_gap_ack_block *gabs)
{
        struct sctp_tsnmap_iter iter;
        int ngaps = 0;

        /* Refresh the gap ack information. */
        if (sctp_tsnmap_has_gap(map)) {
                __u16 start = 0, end = 0;
                sctp_tsnmap_iter_init(map, &iter);
                while (sctp_tsnmap_next_gap_ack(map, &iter,
                                                &start,
                                                &end)) {

                        gabs[ngaps].start = htons(start);
                        gabs[ngaps].end = htons(end);
                        ngaps++;
                        if (ngaps >= SCTP_MAX_GABS)
                                break;
                }
        }
        return ngaps;
}

static int sctp_tsnmap_grow(struct sctp_tsnmap *map, u16 size)
{
        unsigned long *new;
        unsigned long inc;
        u16  len;

        if (size > SCTP_TSN_MAP_SIZE)
                return 0;

        inc = ALIGN((size - map->len), BITS_PER_LONG) + SCTP_TSN_MAP_INCREMENT;
        len = min_t(u16, map->len + inc, SCTP_TSN_MAP_SIZE);

        new = kzalloc(len>>3, GFP_ATOMIC);
        if (!new)
                return 0;

        bitmap_copy(new, map->tsn_map,
                map->max_tsn_seen - map->cumulative_tsn_ack_point);
        kfree(map->tsn_map);
        map->tsn_map = new;
        map->len = len;

        return 1;
}




























































































































































































    4 






















    1 











    1 






    1 






    1 












    1 











    1 















































































































































































































































































































































































































































































































































































































































































































































































































































































































   29 


   10 
    1 


   18 




   27 







































































































































































































































































































































































































   10 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
// SPDX-License-Identifier: GPL-2.0+
/*
 * 2002-10-15  Posix Clocks & timers
 *                           by George Anzinger george@mvista.com
 *                             Copyright (C) 2002 2003 by MontaVista Software.
 *
 * 2004-06-01  Fix CLOCK_REALTIME clock/timer TIMER_ABSTIME bug.
 *                             Copyright (C) 2004 Boris Hu
 *
 * These are all the functions necessary to implement POSIX clocks & timers
 */
#include <linux/compat.h>
#include <linux/compiler.h>
#include <linux/init.h>
#include <linux/jhash.h>
#include <linux/interrupt.h>
#include <linux/list.h>
#include <linux/memblock.h>
#include <linux/nospec.h>
#include <linux/posix-clock.h>
#include <linux/posix-timers.h>
#include <linux/prctl.h>
#include <linux/sched/task.h>
#include <linux/slab.h>
#include <linux/syscalls.h>
#include <linux/time.h>
#include <linux/time_namespace.h>
#include <linux/uaccess.h>

#include "timekeeping.h"
#include "posix-timers.h"

/*
 * Timers are managed in a hash table for lockless lookup. The hash key is
 * constructed from current::signal and the timer ID and the timer is
 * matched against current::signal and the timer ID when walking the hash
 * bucket list.
 *
 * This allows checkpoint/restore to reconstruct the exact timer IDs for
 * a process.
 */
struct timer_hash_bucket {
        spinlock_t                lock;
        struct hlist_head        head;
};

static struct {
        struct timer_hash_bucket        *buckets;
        unsigned long                        mask;
        struct kmem_cache                *cache;
} __timer_data __ro_after_init __aligned(4*sizeof(long));

#define timer_buckets                (__timer_data.buckets)
#define timer_hashmask                (__timer_data.mask)
#define posix_timers_cache        (__timer_data.cache)

static const struct k_clock * const posix_clocks[];
static const struct k_clock *clockid_to_kclock(const clockid_t id);
static const struct k_clock clock_realtime, clock_monotonic;

#define TIMER_ANY_ID                INT_MIN

/* SIGEV_THREAD_ID cannot share a bit with the other SIGEV values. */
#if SIGEV_THREAD_ID != (SIGEV_THREAD_ID & \
                        ~(SIGEV_SIGNAL | SIGEV_NONE | SIGEV_THREAD))
#error "SIGEV_THREAD_ID must not share bit with other SIGEV values!"
#endif

static struct k_itimer *lock_timer(timer_t timer_id);
static inline void unlock_timer(struct k_itimer *timr)
{
        if (likely((timr)))
                spin_unlock_irq(&timr->it_lock);
}

#define scoped_timer_get_or_fail(_id)                                        \
        scoped_cond_guard(lock_timer, return -EINVAL, _id)

#define scoped_timer                                (scope)

DEFINE_CLASS(lock_timer, struct k_itimer *, unlock_timer(_T), lock_timer(id), timer_t id);
DEFINE_CLASS_IS_COND_GUARD(lock_timer);

static struct timer_hash_bucket *hash_bucket(struct signal_struct *sig, unsigned int nr)
{
        return &timer_buckets[jhash2((u32 *)&sig, sizeof(sig) / sizeof(u32), nr) & timer_hashmask];
}

static struct k_itimer *posix_timer_by_id(timer_t id)
{
        struct signal_struct *sig = current->signal;
        struct timer_hash_bucket *bucket = hash_bucket(sig, id);
        struct k_itimer *timer;

        hlist_for_each_entry_rcu(timer, &bucket->head, t_hash) {
                /* timer->it_signal can be set concurrently */
                if ((READ_ONCE(timer->it_signal) == sig) && (timer->it_id == id))
                        return timer;
        }
        return NULL;
}

static inline struct signal_struct *posix_sig_owner(const struct k_itimer *timer)
{
        unsigned long val = (unsigned long)timer->it_signal;

        /*
         * Mask out bit 0, which acts as invalid marker to prevent
         * posix_timer_by_id() detecting it as valid.
         */
        return (struct signal_struct *)(val & ~1UL);
}

static bool posix_timer_hashed(struct timer_hash_bucket *bucket, struct signal_struct *sig,
                               timer_t id)
{
        struct hlist_head *head = &bucket->head;
        struct k_itimer *timer;

        hlist_for_each_entry_rcu(timer, head, t_hash, lockdep_is_held(&bucket->lock)) {
                if ((posix_sig_owner(timer) == sig) && (timer->it_id == id))
                        return true;
        }
        return false;
}

static bool posix_timer_add_at(struct k_itimer *timer, struct signal_struct *sig, unsigned int id)
{
        struct timer_hash_bucket *bucket = hash_bucket(sig, id);

        scoped_guard (spinlock, &bucket->lock) {
                /*
                 * Validate under the lock as this could have raced against
                 * another thread ending up with the same ID, which is
                 * highly unlikely, but possible.
                 */
                if (!posix_timer_hashed(bucket, sig, id)) {
                        /*
                         * Set the timer ID and the signal pointer to make
                         * it identifiable in the hash table. The signal
                         * pointer has bit 0 set to indicate that it is not
                         * yet fully initialized. posix_timer_hashed()
                         * masks this bit out, but the syscall lookup fails
                         * to match due to it being set. This guarantees
                         * that there can't be duplicate timer IDs handed
                         * out.
                         */
                        timer->it_id = (timer_t)id;
                        timer->it_signal = (struct signal_struct *)((unsigned long)sig | 1UL);
                        hlist_add_head_rcu(&timer->t_hash, &bucket->head);
                        return true;
                }
        }
        return false;
}

static int posix_timer_add(struct k_itimer *timer, int req_id)
{
        struct signal_struct *sig = current->signal;

        if (unlikely(req_id != TIMER_ANY_ID)) {
                if (!posix_timer_add_at(timer, sig, req_id))
                        return -EBUSY;

                /*
                 * Move the ID counter past the requested ID, so that after
                 * switching back to normal mode the IDs are outside of the
                 * exact allocated region. That avoids ID collisions on the
                 * next regular timer_create() invocations.
                 */
                atomic_set(&sig->next_posix_timer_id, req_id + 1);
                return req_id;
        }

        for (unsigned int cnt = 0; cnt <= INT_MAX; cnt++) {
                /* Get the next timer ID and clamp it to positive space */
                unsigned int id = atomic_fetch_inc(&sig->next_posix_timer_id) & INT_MAX;

                if (posix_timer_add_at(timer, sig, id))
                        return id;
                cond_resched();
        }
        /* POSIX return code when no timer ID could be allocated */
        return -EAGAIN;
}

static int posix_get_realtime_timespec(clockid_t which_clock, struct timespec64 *tp)
{
        ktime_get_real_ts64(tp);
        return 0;
}

static ktime_t posix_get_realtime_ktime(clockid_t which_clock)
{
        return ktime_get_real();
}

static int posix_clock_realtime_set(const clockid_t which_clock,
                                    const struct timespec64 *tp)
{
        return do_sys_settimeofday64(tp, NULL);
}

static int posix_clock_realtime_adj(const clockid_t which_clock,
                                    struct __kernel_timex *t)
{
        return do_adjtimex(t);
}

static int posix_get_monotonic_timespec(clockid_t which_clock, struct timespec64 *tp)
{
        ktime_get_ts64(tp);
        timens_add_monotonic(tp);
        return 0;
}

static ktime_t posix_get_monotonic_ktime(clockid_t which_clock)
{
        return ktime_get();
}

static int posix_get_monotonic_raw(clockid_t which_clock, struct timespec64 *tp)
{
        ktime_get_raw_ts64(tp);
        timens_add_monotonic(tp);
        return 0;
}

static int posix_get_realtime_coarse(clockid_t which_clock, struct timespec64 *tp)
{
        ktime_get_coarse_real_ts64(tp);
        return 0;
}

static int posix_get_monotonic_coarse(clockid_t which_clock,
                                                struct timespec64 *tp)
{
        ktime_get_coarse_ts64(tp);
        timens_add_monotonic(tp);
        return 0;
}

static int posix_get_coarse_res(const clockid_t which_clock, struct timespec64 *tp)
{
        *tp = ktime_to_timespec64(KTIME_LOW_RES);
        return 0;
}

static int posix_get_boottime_timespec(const clockid_t which_clock, struct timespec64 *tp)
{
        ktime_get_boottime_ts64(tp);
        timens_add_boottime(tp);
        return 0;
}

static ktime_t posix_get_boottime_ktime(const clockid_t which_clock)
{
        return ktime_get_boottime();
}

static int posix_get_tai_timespec(clockid_t which_clock, struct timespec64 *tp)
{
        ktime_get_clocktai_ts64(tp);
        return 0;
}

static ktime_t posix_get_tai_ktime(clockid_t which_clock)
{
        return ktime_get_clocktai();
}

static int posix_get_hrtimer_res(clockid_t which_clock, struct timespec64 *tp)
{
        tp->tv_sec = 0;
        tp->tv_nsec = hrtimer_resolution;
        return 0;
}

/*
 * The siginfo si_overrun field and the return value of timer_getoverrun(2)
 * are of type int. Clamp the overrun value to INT_MAX
 */
static inline int timer_overrun_to_int(struct k_itimer *timr)
{
        if (timr->it_overrun_last > (s64)INT_MAX)
                return INT_MAX;

        return (int)timr->it_overrun_last;
}

static bool common_hrtimer_rearm(struct k_itimer *timr)
{
        struct hrtimer *timer = &timr->it.real.timer;

        timr->it_overrun += hrtimer_forward_now(timer, timr->it_interval);
        return hrtimer_start_expires_user(timer, HRTIMER_MODE_ABS);
}

static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_itimer *timr)
{
        bool queued;

        guard(spinlock)(&timr->it_lock);

        /*
         * Check if the timer is still alive or whether it got modified
         * since the signal was queued. In either case, don't rearm and
         * drop the signal.
         */
        if (timr->it_signal_seq != timr->it_sigqueue_seq || WARN_ON_ONCE(!posixtimer_valid(timr)))
                return false;

        if (!timr->it_interval || WARN_ON_ONCE(timr->it_status != POSIX_TIMER_REQUEUE_PENDING))
                return true;

        /* timer_rearm() updates timr::it_overrun */
        queued = timr->kclock->timer_rearm(timr);

        timr->it_overrun_last = timr->it_overrun;
        timr->it_overrun = -1LL;
        ++timr->it_signal_seq;
        info->si_overrun = timer_overrun_to_int(timr);

        if (queued)
                timr->it_status = POSIX_TIMER_ARMED;
        else
                posix_timer_queue_signal(timr);
        return true;
}

/*
 * This function is called from the signal delivery code. It decides
 * whether the signal should be dropped and rearms interval timers.  The
 * timer can be unconditionally accessed as there is a reference held on
 * it.
 */
bool posixtimer_deliver_signal(struct kernel_siginfo *info, struct sigqueue *timer_sigq)
{
        struct k_itimer *timr = container_of(timer_sigq, struct k_itimer, sigq);
        bool ret;

        /*
         * Release siglock to ensure proper locking order versus
         * timr::it_lock. Keep interrupts disabled.
         */
        spin_unlock(&current->sighand->siglock);

        ret = __posixtimer_deliver_signal(info, timr);

        /* Drop the reference which was acquired when the signal was queued */
        posixtimer_putref(timr);

        spin_lock(&current->sighand->siglock);
        return ret;
}

void posix_timer_queue_signal(struct k_itimer *timr)
{
        lockdep_assert_held(&timr->it_lock);

        if (!posixtimer_valid(timr))
                return;

        timr->it_status = timr->it_interval ? POSIX_TIMER_REQUEUE_PENDING : POSIX_TIMER_DISARMED;
        posixtimer_send_sigqueue(timr);
}

/*
 * This function gets called when a POSIX.1b interval timer expires from
 * the HRTIMER interrupt (soft interrupt on RT kernels).
 *
 * Handles CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_BOOTTIME and CLOCK_TAI
 * based timers.
 */
static enum hrtimer_restart posix_timer_fn(struct hrtimer *timer)
{
        struct k_itimer *timr = container_of(timer, struct k_itimer, it.real.timer);

        guard(spinlock_irqsave)(&timr->it_lock);
        posix_timer_queue_signal(timr);
        return HRTIMER_NORESTART;
}

long posixtimer_create_prctl(unsigned long ctrl)
{
        switch (ctrl) {
        case PR_TIMER_CREATE_RESTORE_IDS_OFF:
                current->signal->timer_create_restore_ids = 0;
                return 0;
        case PR_TIMER_CREATE_RESTORE_IDS_ON:
                current->signal->timer_create_restore_ids = 1;
                return 0;
        case PR_TIMER_CREATE_RESTORE_IDS_GET:
                return current->signal->timer_create_restore_ids;
        }
        return -EINVAL;
}

static struct pid *good_sigevent(sigevent_t * event)
{
        struct pid *pid = task_tgid(current);
        struct task_struct *rtn;

        switch (event->sigev_notify) {
        case SIGEV_SIGNAL | SIGEV_THREAD_ID:
                pid = find_vpid(event->sigev_notify_thread_id);
                rtn = pid_task(pid, PIDTYPE_PID);
                if (!rtn || !same_thread_group(rtn, current))
                        return NULL;
                fallthrough;
        case SIGEV_SIGNAL:
        case SIGEV_THREAD:
                if (event->sigev_signo <= 0 || event->sigev_signo > SIGRTMAX)
                        return NULL;
                fallthrough;
        case SIGEV_NONE:
                return pid;
        default:
                return NULL;
        }
}

static struct k_itimer *alloc_posix_timer(void)
{
        struct k_itimer *tmr;

        if (unlikely(!posix_timers_cache))
                return NULL;

        tmr = kmem_cache_zalloc(posix_timers_cache, GFP_KERNEL);
        if (!tmr)
                return tmr;

        if (unlikely(!posixtimer_init_sigqueue(&tmr->sigq))) {
                kmem_cache_free(posix_timers_cache, tmr);
                return NULL;
        }
        rcuref_init(&tmr->rcuref, 1);
        return tmr;
}

void posixtimer_free_timer(struct k_itimer *tmr)
{
        put_pid(tmr->it_pid);
        if (tmr->sigq.ucounts)
                dec_rlimit_put_ucounts(tmr->sigq.ucounts, UCOUNT_RLIMIT_SIGPENDING);
        kfree_rcu(tmr, rcu);
}

static void posix_timer_unhash_and_free(struct k_itimer *tmr)
{
        struct timer_hash_bucket *bucket = hash_bucket(posix_sig_owner(tmr), tmr->it_id);

        scoped_guard (spinlock, &bucket->lock)
                hlist_del_rcu(&tmr->t_hash);
        posixtimer_putref(tmr);
}

static int common_timer_create(struct k_itimer *new_timer)
{
        hrtimer_setup(&new_timer->it.real.timer, posix_timer_fn, new_timer->it_clock, 0);
        return 0;
}

/* Create a POSIX.1b interval timer. */
static int do_timer_create(clockid_t which_clock, struct sigevent *event,
                           timer_t __user *created_timer_id)
{
        const struct k_clock *kc = clockid_to_kclock(which_clock);
        timer_t req_id = TIMER_ANY_ID;
        struct k_itimer *new_timer;
        int error, new_timer_id;

        if (!kc)
                return -EINVAL;
        if (!kc->timer_create)
                return -EOPNOTSUPP;

        /* Special case for CRIU to restore timers with a given timer ID. */
        if (unlikely(current->signal->timer_create_restore_ids)) {
                if (copy_from_user(&req_id, created_timer_id, sizeof(req_id)))
                        return -EFAULT;
                /* Valid IDs are 0..INT_MAX */
                if ((unsigned int)req_id > INT_MAX)
                        return -EINVAL;
        }

        new_timer = alloc_posix_timer();
        if (unlikely(!new_timer))
                return -EAGAIN;

        spin_lock_init(&new_timer->it_lock);

        /*
         * Add the timer to the hash table. The timer is not yet valid
         * after insertion, but has a unique ID allocated.
         */
        new_timer_id = posix_timer_add(new_timer, req_id);
        if (new_timer_id < 0) {
                posixtimer_free_timer(new_timer);
                return new_timer_id;
        }

        new_timer->it_clock = which_clock;
        new_timer->kclock = kc;
        new_timer->it_overrun = -1LL;

        if (event) {
                scoped_guard (rcu)
                        new_timer->it_pid = get_pid(good_sigevent(event));
                if (!new_timer->it_pid) {
                        error = -EINVAL;
                        goto out;
                }
                new_timer->it_sigev_notify     = event->sigev_notify;
                new_timer->sigq.info.si_signo = event->sigev_signo;
                new_timer->sigq.info.si_value = event->sigev_value;
        } else {
                new_timer->it_sigev_notify     = SIGEV_SIGNAL;
                new_timer->sigq.info.si_signo = SIGALRM;
                new_timer->sigq.info.si_value.sival_int = new_timer->it_id;
                new_timer->it_pid = get_pid(task_tgid(current));
        }

        if (new_timer->it_sigev_notify & SIGEV_THREAD_ID)
                new_timer->it_pid_type = PIDTYPE_PID;
        else
                new_timer->it_pid_type = PIDTYPE_TGID;

        new_timer->sigq.info.si_tid = new_timer->it_id;
        new_timer->sigq.info.si_code = SI_TIMER;

        if (copy_to_user(created_timer_id, &new_timer_id, sizeof (new_timer_id))) {
                error = -EFAULT;
                goto out;
        }
        /*
         * After successful copy out, the timer ID is visible to user space
         * now but not yet valid because new_timer::signal low order bit is 1.
         *
         * Complete the initialization with the clock specific create
         * callback.
         */
        error = kc->timer_create(new_timer);
        if (error)
                goto out;

        /*
         * timer::it_lock ensures that __lock_timer() observes a fully
         * initialized timer when it observes a valid timer::it_signal.
         *
         * sighand::siglock is required to protect signal::posix_timers.
         */
        scoped_guard (spinlock_irq, &new_timer->it_lock) {
                guard(spinlock)(&current->sighand->siglock);
                /*
                 * new_timer::it_signal contains the signal pointer with
                 * bit 0 set, which makes it invalid for syscall operations.
                 * Store the unmodified signal pointer to make it valid.
                 */
                WRITE_ONCE(new_timer->it_signal, current->signal);
                hlist_add_head_rcu(&new_timer->list, &current->signal->posix_timers);
        }
        /*
         * After unlocking @new_timer is subject to concurrent removal and
         * cannot be touched anymore
         */
        return 0;
out:
        posix_timer_unhash_and_free(new_timer);
        return error;
}

SYSCALL_DEFINE3(timer_create, const clockid_t, which_clock,
                struct sigevent __user *, timer_event_spec,
                timer_t __user *, created_timer_id)
{
        if (timer_event_spec) {
                sigevent_t event;

                if (copy_from_user(&event, timer_event_spec, sizeof (event)))
                        return -EFAULT;
                return do_timer_create(which_clock, &event, created_timer_id);
        }
        return do_timer_create(which_clock, NULL, created_timer_id);
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE3(timer_create, clockid_t, which_clock,
                       struct compat_sigevent __user *, timer_event_spec,
                       timer_t __user *, created_timer_id)
{
        if (timer_event_spec) {
                sigevent_t event;

                if (get_compat_sigevent(&event, timer_event_spec))
                        return -EFAULT;
                return do_timer_create(which_clock, &event, created_timer_id);
        }
        return do_timer_create(which_clock, NULL, created_timer_id);
}
#endif

static struct k_itimer *lock_timer(timer_t timer_id)
{
        struct k_itimer *timr;

        /*
         * timer_t could be any type >= int and we want to make sure any
         * @timer_id outside positive int range fails lookup.
         */
        if ((unsigned long long)timer_id > INT_MAX)
                return NULL;

        /*
         * The hash lookup and the timers are RCU protected.
         *
         * Timers are added to the hash in invalid state where
         * timr::it_signal is marked invalid. timer::it_signal is only set
         * after the rest of the initialization succeeded.
         *
         * Timer destruction happens in steps:
         *  1) Set timr::it_signal marked invalid with timr::it_lock held
         *  2) Release timr::it_lock
         *  3) Remove from the hash under hash_lock
         *  4) Put the reference count.
         *
         * The reference count might not drop to zero if timr::sigq is
         * queued. In that case the signal delivery or flush will put the
         * last reference count.
         *
         * When the reference count reaches zero, the timer is scheduled
         * for RCU removal after the grace period.
         *
         * Holding rcu_read_lock() across the lookup ensures that
         * the timer cannot be freed.
         *
         * The lookup validates locklessly that timr::it_signal ==
         * current::it_signal and timr::it_id == @timer_id. timr::it_id
         * can't change, but timr::it_signal can become invalid during
         * destruction, which makes the locked check fail.
         */
        guard(rcu)();
        timr = posix_timer_by_id(timer_id);
        if (timr) {
                spin_lock_irq(&timr->it_lock);
                /*
                 * Validate under timr::it_lock that timr::it_signal is
                 * still valid. Pairs with #1 above.
                 */
                if (timr->it_signal == current->signal)
                        return timr;
                spin_unlock_irq(&timr->it_lock);
        }
        return NULL;
}

static ktime_t common_hrtimer_remaining(struct k_itimer *timr, ktime_t now)
{
        struct hrtimer *timer = &timr->it.real.timer;

        return __hrtimer_expires_remaining_adjusted(timer, now);
}

static s64 common_hrtimer_forward(struct k_itimer *timr, ktime_t now)
{
        struct hrtimer *timer = &timr->it.real.timer;

        return hrtimer_forward(timer, now, timr->it_interval);
}

/*
 * Get the time remaining on a POSIX.1b interval timer.
 *
 * Two issues to handle here:
 *
 *  1) The timer has a requeue pending. The return value must appear as
 *     if the timer has been requeued right now.
 *
 *  2) The timer is a SIGEV_NONE timer. These timers are never enqueued
 *     into the hrtimer queue and therefore never expired. Emulate expiry
 *     here taking #1 into account.
 */
void common_timer_get(struct k_itimer *timr, struct itimerspec64 *cur_setting)
{
        const struct k_clock *kc = timr->kclock;
        ktime_t now, remaining, iv;
        bool sig_none;

        sig_none = timr->it_sigev_notify == SIGEV_NONE;
        iv = timr->it_interval;

        /* interval timer ? */
        if (iv) {
                cur_setting->it_interval = ktime_to_timespec64(iv);
        } else if (timr->it_status == POSIX_TIMER_DISARMED) {
                /*
                 * SIGEV_NONE oneshot timers are never queued and therefore
                 * timr->it_status is always DISARMED. The check below
                 * vs. remaining time will handle this case.
                 *
                 * For all other timers there is nothing to update here, so
                 * return.
                 */
                if (!sig_none)
                        return;
        }

        now = kc->clock_get_ktime(timr->it_clock);

        /*
         * If this is an interval timer and either has requeue pending or
         * is a SIGEV_NONE timer move the expiry time forward by intervals,
         * so expiry is > now.
         */
        if (iv && timr->it_status != POSIX_TIMER_ARMED)
                timr->it_overrun += kc->timer_forward(timr, now);

        remaining = kc->timer_remaining(timr, now);
        /*
         * As @now is retrieved before a possible timer_forward() and
         * cannot be reevaluated by the compiler @remaining is based on the
         * same @now value. Therefore @remaining is consistent vs. @now.
         *
         * Consequently all interval timers, i.e. @iv > 0, cannot have a
         * remaining time <= 0 because timer_forward() guarantees to move
         * them forward so that the next timer expiry is > @now.
         */
        if (remaining <= 0) {
                /*
                 * A single shot SIGEV_NONE timer must return 0, when it is
                 * expired! Timers which have a real signal delivery mode
                 * must return a remaining time greater than 0 because the
                 * signal has not yet been delivered.
                 */
                if (!sig_none)
                        cur_setting->it_value.tv_nsec = 1;
        } else {
                cur_setting->it_value = ktime_to_timespec64(remaining);
        }
}

static int do_timer_gettime(timer_t timer_id,  struct itimerspec64 *setting)
{
        memset(setting, 0, sizeof(*setting));
        scoped_timer_get_or_fail(timer_id)
                scoped_timer->kclock->timer_get(scoped_timer, setting);
        return 0;
}

/* Get the time remaining on a POSIX.1b interval timer. */
SYSCALL_DEFINE2(timer_gettime, timer_t, timer_id,
                struct __kernel_itimerspec __user *, setting)
{
        struct itimerspec64 cur_setting;

        int ret = do_timer_gettime(timer_id, &cur_setting);
        if (!ret) {
                if (put_itimerspec64(&cur_setting, setting))
                        ret = -EFAULT;
        }
        return ret;
}

#ifdef CONFIG_COMPAT_32BIT_TIME

SYSCALL_DEFINE2(timer_gettime32, timer_t, timer_id,
                struct old_itimerspec32 __user *, setting)
{
        struct itimerspec64 cur_setting;

        int ret = do_timer_gettime(timer_id, &cur_setting);
        if (!ret) {
                if (put_old_itimerspec32(&cur_setting, setting))
                        ret = -EFAULT;
        }
        return ret;
}

#endif

/**
 * sys_timer_getoverrun - Get the number of overruns of a POSIX.1b interval timer
 * @timer_id:        The timer ID which identifies the timer
 *
 * The "overrun count" of a timer is one plus the number of expiration
 * intervals which have elapsed between the first expiry, which queues the
 * signal and the actual signal delivery. On signal delivery the "overrun
 * count" is calculated and cached, so it can be returned directly here.
 *
 * As this is relative to the last queued signal the returned overrun count
 * is meaningless outside of the signal delivery path and even there it
 * does not accurately reflect the current state when user space evaluates
 * it.
 *
 * Returns:
 *        -EINVAL                @timer_id is invalid
 *        1..INT_MAX        The number of overruns related to the last delivered signal
 */
SYSCALL_DEFINE1(timer_getoverrun, timer_t, timer_id)
{
        scoped_timer_get_or_fail(timer_id)
                return timer_overrun_to_int(scoped_timer);
}

static bool common_hrtimer_arm(struct k_itimer *timr, ktime_t expires,
                               bool absolute, bool sigev_none)
{
        struct hrtimer *timer = &timr->it.real.timer;
        enum hrtimer_mode mode;

        mode = absolute ? HRTIMER_MODE_ABS : HRTIMER_MODE_REL;
        /*
         * Posix magic: Relative CLOCK_REALTIME timers are not affected by
         * clock modifications, so they become CLOCK_MONOTONIC based under the
         * hood. See hrtimer_setup(). Update timr->kclock, so the generic
         * functions which use timr->kclock->clock_get_*() work.
         *
         * Note: it_clock stays unmodified, because the next timer_set() might
         * use ABSTIME, so it needs to switch back.
         */
        if (timr->it_clock == CLOCK_REALTIME)
                timr->kclock = absolute ? &clock_realtime : &clock_monotonic;

        hrtimer_setup(&timr->it.real.timer, posix_timer_fn, timr->it_clock, mode);

        if (!absolute)
                expires = ktime_add_safe(expires, hrtimer_cb_get_time(timer));
        hrtimer_set_expires(timer, expires);

        /* For sigev_none pretend that the timer is queued */
        if (sigev_none)
                return true;

        return hrtimer_start_expires_user(timer, HRTIMER_MODE_ABS);
}

static int common_hrtimer_try_to_cancel(struct k_itimer *timr)
{
        return hrtimer_try_to_cancel(&timr->it.real.timer);
}

static void common_timer_wait_running(struct k_itimer *timer)
{
        hrtimer_cancel_wait_running(&timer->it.real.timer);
}

/*
 * On PREEMPT_RT this prevents priority inversion and a potential livelock
 * against the ksoftirqd thread in case that ksoftirqd gets preempted while
 * executing a hrtimer callback.
 *
 * See the comments in hrtimer_cancel_wait_running(). For PREEMPT_RT=n this
 * just results in a cpu_relax().
 *
 * For POSIX CPU timers with CONFIG_POSIX_CPU_TIMERS_TASK_WORK=n this is
 * just a cpu_relax(). With CONFIG_POSIX_CPU_TIMERS_TASK_WORK=y this
 * prevents spinning on an eventually scheduled out task and a livelock
 * when the task which tries to delete or disarm the timer has preempted
 * the task which runs the expiry in task work context.
 */
static void timer_wait_running(struct k_itimer *timer)
{
        /*
         * kc->timer_wait_running() might drop RCU lock. So @timer
         * cannot be touched anymore after the function returns!
         */
        timer->kclock->timer_wait_running(timer);
}

/*
 * Set up the new interval and reset the signal delivery data
 */
void posix_timer_set_common(struct k_itimer *timer, struct itimerspec64 *new_setting)
{
        if (new_setting->it_value.tv_sec || new_setting->it_value.tv_nsec)
                timer->it_interval = timespec64_to_ktime(new_setting->it_interval);
        else
                timer->it_interval = 0;

        /* Reset overrun accounting */
        timer->it_overrun_last = 0;
        timer->it_overrun = -1LL;
}

/* Set a POSIX.1b interval timer. */
int common_timer_set(struct k_itimer *timr, int flags,
                     struct itimerspec64 *new_setting,
                     struct itimerspec64 *old_setting)
{
        const struct k_clock *kc = timr->kclock;
        bool sigev_none;
        ktime_t expires;

        if (old_setting)
                common_timer_get(timr, old_setting);

        /*
         * Careful here. On SMP systems the timer expiry function could be
         * active and spinning on timr->it_lock.
         */
        if (kc->timer_try_to_cancel(timr) < 0)
                return TIMER_RETRY;

        timr->it_status = POSIX_TIMER_DISARMED;
        posix_timer_set_common(timr, new_setting);

        /* Keep timer disarmed when it_value is zero */
        if (!new_setting->it_value.tv_sec && !new_setting->it_value.tv_nsec)
                return 0;

        expires = timespec64_to_ktime(new_setting->it_value);
        if (flags & TIMER_ABSTIME)
                expires = timens_ktime_to_host(timr->it_clock, expires);
        sigev_none = timr->it_sigev_notify == SIGEV_NONE;

        if (kc->timer_arm(timr, expires, flags & TIMER_ABSTIME, sigev_none)) {
                if (!sigev_none)
                        timr->it_status = POSIX_TIMER_ARMED;
        } else {
                /* Timer was already expired, queue the signal */
                posix_timer_queue_signal(timr);
        }
        return 0;
}

static int do_timer_settime(timer_t timer_id, int tmr_flags, struct itimerspec64 *new_spec64,
                            struct itimerspec64 *old_spec64)
{
        if (!timespec64_valid(&new_spec64->it_interval) ||
            !timespec64_valid(&new_spec64->it_value))
                return -EINVAL;

        if (old_spec64)
                memset(old_spec64, 0, sizeof(*old_spec64));

        for (; ; old_spec64 = NULL) {
                struct k_itimer *timr;

                scoped_timer_get_or_fail(timer_id) {
                        timr = scoped_timer;

                        if (old_spec64)
                                old_spec64->it_interval = ktime_to_timespec64(timr->it_interval);

                        /* Prevent signal delivery and rearming. */
                        timr->it_signal_seq++;

                        int ret = timr->kclock->timer_set(timr, tmr_flags, new_spec64, old_spec64);
                        if (ret != TIMER_RETRY)
                                return ret;

                        /* Protect the timer from being freed when leaving the lock scope */
                        rcu_read_lock();
                }
                timer_wait_running(timr);
                rcu_read_unlock();
        }
}

/* Set a POSIX.1b interval timer */
SYSCALL_DEFINE4(timer_settime, timer_t, timer_id, int, flags,
                const struct __kernel_itimerspec __user *, new_setting,
                struct __kernel_itimerspec __user *, old_setting)
{
        struct itimerspec64 new_spec, old_spec, *rtn;
        int error = 0;

        if (!new_setting)
                return -EINVAL;

        if (get_itimerspec64(&new_spec, new_setting))
                return -EFAULT;

        rtn = old_setting ? &old_spec : NULL;
        error = do_timer_settime(timer_id, flags, &new_spec, rtn);
        if (!error && old_setting) {
                if (put_itimerspec64(&old_spec, old_setting))
                        error = -EFAULT;
        }
        return error;
}

#ifdef CONFIG_COMPAT_32BIT_TIME
SYSCALL_DEFINE4(timer_settime32, timer_t, timer_id, int, flags,
                struct old_itimerspec32 __user *, new,
                struct old_itimerspec32 __user *, old)
{
        struct itimerspec64 new_spec, old_spec;
        struct itimerspec64 *rtn = old ? &old_spec : NULL;
        int error = 0;

        if (!new)
                return -EINVAL;
        if (get_old_itimerspec32(&new_spec, new))
                return -EFAULT;

        error = do_timer_settime(timer_id, flags, &new_spec, rtn);
        if (!error && old) {
                if (put_old_itimerspec32(&old_spec, old))
                        error = -EFAULT;
        }
        return error;
}
#endif

int common_timer_del(struct k_itimer *timer)
{
        const struct k_clock *kc = timer->kclock;

        if (kc->timer_try_to_cancel(timer) < 0)
                return TIMER_RETRY;
        timer->it_status = POSIX_TIMER_DISARMED;
        return 0;
}

/*
 * If the deleted timer is on the ignored list, remove it and
 * drop the associated reference.
 */
static inline void posix_timer_cleanup_ignored(struct k_itimer *tmr)
{
        if (!hlist_unhashed(&tmr->ignored_list)) {
                hlist_del_init(&tmr->ignored_list);
                posixtimer_putref(tmr);
        }
}

static void posix_timer_delete(struct k_itimer *timer)
{
        /*
         * Invalidate the timer, remove it from the linked list and remove
         * it from the ignored list if pending.
         *
         * The invalidation must be written with siglock held so that the
         * signal code observes the invalidated timer::it_signal in
         * do_sigaction(), which prevents it from moving a pending signal
         * of a deleted timer to the ignore list.
         *
         * The invalidation also prevents signal queueing, signal delivery
         * and therefore rearming from the signal delivery path.
         *
         * A concurrent lookup can still find the timer in the hash, but it
         * will check timer::it_signal with timer::it_lock held and observe
         * bit 0 set, which invalidates it. That also prevents the timer ID
         * from being handed out before this timer is completely gone.
         */
        timer->it_signal_seq++;

        scoped_guard (spinlock, &current->sighand->siglock) {
                unsigned long sig = (unsigned long)timer->it_signal | 1UL;

                WRITE_ONCE(timer->it_signal, (struct signal_struct *)sig);
                hlist_del_rcu(&timer->list);
                posix_timer_cleanup_ignored(timer);
        }

        while (timer->kclock->timer_del(timer) == TIMER_RETRY) {
                guard(rcu)();
                spin_unlock_irq(&timer->it_lock);
                timer_wait_running(timer);
                spin_lock_irq(&timer->it_lock);
        }
}

/* Delete a POSIX.1b interval timer. */
SYSCALL_DEFINE1(timer_delete, timer_t, timer_id)
{
        struct k_itimer *timer;

        scoped_timer_get_or_fail(timer_id) {
                timer = scoped_timer;
                posix_timer_delete(timer);
        }
        /* Remove it from the hash, which frees up the timer ID */
        posix_timer_unhash_and_free(timer);
        return 0;
}

/*
 * Invoked from do_exit() when the last thread of a thread group exits.
 * At that point no other task can access the timers of the dying
 * task anymore.
 */
void exit_itimers(struct task_struct *tsk)
{
        struct hlist_head timers;
        struct hlist_node *next;
        struct k_itimer *timer;

        /* Clear restore mode for exec() */
        tsk->signal->timer_create_restore_ids = 0;

        if (hlist_empty(&tsk->signal->posix_timers))
                return;

        /* Protect against concurrent read via /proc/$PID/timers */
        scoped_guard (spinlock_irq, &tsk->sighand->siglock)
                hlist_move_list(&tsk->signal->posix_timers, &timers);

        /* The timers are not longer accessible via tsk::signal */
        hlist_for_each_entry_safe(timer, next, &timers, list) {
                scoped_guard (spinlock_irq, &timer->it_lock)
                        posix_timer_delete(timer);
                posix_timer_unhash_and_free(timer);
                cond_resched();
        }

        /*
         * There should be no timers on the ignored list. posix_timer_delete() has
         * mopped them up.
         */
        if (!WARN_ON_ONCE(!hlist_empty(&tsk->signal->ignored_posix_timers)))
                return;

        hlist_move_list(&tsk->signal->ignored_posix_timers, &timers);
        while (!hlist_empty(&timers)) {
                posix_timer_cleanup_ignored(hlist_entry(timers.first, struct k_itimer,
                                                        ignored_list));
        }
}

SYSCALL_DEFINE2(clock_settime, const clockid_t, which_clock,
                const struct __kernel_timespec __user *, tp)
{
        const struct k_clock *kc = clockid_to_kclock(which_clock);
        struct timespec64 new_tp;

        if (!kc || !kc->clock_set)
                return -EINVAL;

        if (get_timespec64(&new_tp, tp))
                return -EFAULT;

        /*
         * Permission checks have to be done inside the clock specific
         * setter callback.
         */
        return kc->clock_set(which_clock, &new_tp);
}

SYSCALL_DEFINE2(clock_gettime, const clockid_t, which_clock,
                struct __kernel_timespec __user *, tp)
{
        const struct k_clock *kc = clockid_to_kclock(which_clock);
        struct timespec64 kernel_tp;
        int error;

        if (!kc)
                return -EINVAL;

        error = kc->clock_get_timespec(which_clock, &kernel_tp);

        if (!error && put_timespec64(&kernel_tp, tp))
                error = -EFAULT;

        return error;
}

int do_clock_adjtime(const clockid_t which_clock, struct __kernel_timex * ktx)
{
        const struct k_clock *kc = clockid_to_kclock(which_clock);

        if (!kc)
                return -EINVAL;
        if (!kc->clock_adj)
                return -EOPNOTSUPP;

        return kc->clock_adj(which_clock, ktx);
}

SYSCALL_DEFINE2(clock_adjtime, const clockid_t, which_clock,
                struct __kernel_timex __user *, utx)
{
        struct __kernel_timex ktx;
        int err;

        if (copy_from_user(&ktx, utx, sizeof(ktx)))
                return -EFAULT;

        err = do_clock_adjtime(which_clock, &ktx);

        if (err >= 0 && copy_to_user(utx, &ktx, sizeof(ktx)))
                return -EFAULT;

        return err;
}

/**
 * sys_clock_getres - Get the resolution of a clock
 * @which_clock:        The clock to get the resolution for
 * @tp:                        Pointer to a a user space timespec64 for storage
 *
 * POSIX defines:
 *
 * "The clock_getres() function shall return the resolution of any
 * clock. Clock resolutions are implementation-defined and cannot be set by
 * a process. If the argument res is not NULL, the resolution of the
 * specified clock shall be stored in the location pointed to by res. If
 * res is NULL, the clock resolution is not returned. If the time argument
 * of clock_settime() is not a multiple of res, then the value is truncated
 * to a multiple of res."
 *
 * Due to the various hardware constraints the real resolution can vary
 * wildly and even change during runtime when the underlying devices are
 * replaced. The kernel also can use hardware devices with different
 * resolutions for reading the time and for arming timers.
 *
 * The kernel therefore deviates from the POSIX spec in various aspects:
 *
 * 1) The resolution returned to user space
 *
 *    For CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_BOOTTIME, CLOCK_TAI,
 *    CLOCK_REALTIME_ALARM, CLOCK_BOOTTIME_ALAREM and CLOCK_MONOTONIC_RAW
 *    the kernel differentiates only two cases:
 *
 *    I)  Low resolution mode:
 *
 *          When high resolution timers are disabled at compile or runtime
 *          the resolution returned is nanoseconds per tick, which represents
 *          the precision at which timers expire.
 *
 *    II) High resolution mode:
 *
 *          When high resolution timers are enabled the resolution returned
 *          is always one nanosecond independent of the actual resolution of
 *          the underlying hardware devices.
 *
 *          For CLOCK_*_ALARM the actual resolution depends on system
 *          state. When system is running the resolution is the same as the
 *          resolution of the other clocks. During suspend the actual
 *          resolution is the resolution of the underlying RTC device which
 *          might be way less precise than the clockevent device used during
 *          running state.
 *
 *   For CLOCK_REALTIME_COARSE and CLOCK_MONOTONIC_COARSE the resolution
 *   returned is always nanoseconds per tick.
 *
 *   For CLOCK_PROCESS_CPUTIME and CLOCK_THREAD_CPUTIME the resolution
 *   returned is always one nanosecond under the assumption that the
 *   underlying scheduler clock has a better resolution than nanoseconds
 *   per tick.
 *
 *   For dynamic POSIX clocks (PTP devices) the resolution returned is
 *   always one nanosecond.
 *
 * 2) Affect on sys_clock_settime()
 *
 *    The kernel does not truncate the time which is handed in to
 *    sys_clock_settime(). The kernel internal timekeeping is always using
 *    nanoseconds precision independent of the clocksource device which is
 *    used to read the time from. The resolution of that device only
 *    affects the precision of the time returned by sys_clock_gettime().
 *
 * Returns:
 *        0                Success. @tp contains the resolution
 *        -EINVAL                @which_clock is not a valid clock ID
 *        -EFAULT                Copying the resolution to @tp faulted
 *        -ENODEV                Dynamic POSIX clock is not backed by a device
 *        -EOPNOTSUPP        Dynamic POSIX clock does not support getres()
 */
SYSCALL_DEFINE2(clock_getres, const clockid_t, which_clock,
                struct __kernel_timespec __user *, tp)
{
        const struct k_clock *kc = clockid_to_kclock(which_clock);
        struct timespec64 rtn_tp;
        int error;

        if (!kc)
                return -EINVAL;

        error = kc->clock_getres(which_clock, &rtn_tp);

        if (!error && tp && put_timespec64(&rtn_tp, tp))
                error = -EFAULT;

        return error;
}

#ifdef CONFIG_COMPAT_32BIT_TIME

SYSCALL_DEFINE2(clock_settime32, clockid_t, which_clock,
                struct old_timespec32 __user *, tp)
{
        const struct k_clock *kc = clockid_to_kclock(which_clock);
        struct timespec64 ts;

        if (!kc || !kc->clock_set)
                return -EINVAL;

        if (get_old_timespec32(&ts, tp))
                return -EFAULT;

        return kc->clock_set(which_clock, &ts);
}

SYSCALL_DEFINE2(clock_gettime32, clockid_t, which_clock,
                struct old_timespec32 __user *, tp)
{
        const struct k_clock *kc = clockid_to_kclock(which_clock);
        struct timespec64 ts;
        int err;

        if (!kc)
                return -EINVAL;

        err = kc->clock_get_timespec(which_clock, &ts);

        if (!err && put_old_timespec32(&ts, tp))
                err = -EFAULT;

        return err;
}

SYSCALL_DEFINE2(clock_adjtime32, clockid_t, which_clock,
                struct old_timex32 __user *, utp)
{
        struct __kernel_timex ktx;
        int err;

        err = get_old_timex32(&ktx, utp);
        if (err)
                return err;

        err = do_clock_adjtime(which_clock, &ktx);

        if (err >= 0 && put_old_timex32(utp, &ktx))
                return -EFAULT;

        return err;
}

SYSCALL_DEFINE2(clock_getres_time32, clockid_t, which_clock,
                struct old_timespec32 __user *, tp)
{
        const struct k_clock *kc = clockid_to_kclock(which_clock);
        struct timespec64 ts;
        int err;

        if (!kc)
                return -EINVAL;

        err = kc->clock_getres(which_clock, &ts);
        if (!err && tp && put_old_timespec32(&ts, tp))
                return -EFAULT;

        return err;
}

#endif

/*
 * sys_clock_nanosleep() for CLOCK_REALTIME and CLOCK_TAI
 */
static int common_nsleep(const clockid_t which_clock, int flags,
                         const struct timespec64 *rqtp)
{
        ktime_t texp = timespec64_to_ktime(*rqtp);

        return hrtimer_nanosleep(texp, flags & TIMER_ABSTIME ?
                                 HRTIMER_MODE_ABS : HRTIMER_MODE_REL,
                                 which_clock);
}

/*
 * sys_clock_nanosleep() for CLOCK_MONOTONIC and CLOCK_BOOTTIME
 *
 * Absolute nanosleeps for these clocks are time-namespace adjusted.
 */
static int common_nsleep_timens(const clockid_t which_clock, int flags,
                                const struct timespec64 *rqtp)
{
        ktime_t texp = timespec64_to_ktime(*rqtp);

        if (flags & TIMER_ABSTIME)
                texp = timens_ktime_to_host(which_clock, texp);

        return hrtimer_nanosleep(texp, flags & TIMER_ABSTIME ?
                                 HRTIMER_MODE_ABS : HRTIMER_MODE_REL,
                                 which_clock);
}

SYSCALL_DEFINE4(clock_nanosleep, const clockid_t, which_clock, int, flags,
                const struct __kernel_timespec __user *, rqtp,
                struct __kernel_timespec __user *, rmtp)
{
        const struct k_clock *kc = clockid_to_kclock(which_clock);
        struct timespec64 t;

        if (!kc)
                return -EINVAL;
        if (!kc->nsleep)
                return -EOPNOTSUPP;

        if (get_timespec64(&t, rqtp))
                return -EFAULT;

        if (!timespec64_valid(&t))
                return -EINVAL;
        if (flags & TIMER_ABSTIME)
                rmtp = NULL;
        current->restart_block.fn = do_no_restart_syscall;
        current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
        current->restart_block.nanosleep.rmtp = rmtp;

        return kc->nsleep(which_clock, flags, &t);
}

#ifdef CONFIG_COMPAT_32BIT_TIME

SYSCALL_DEFINE4(clock_nanosleep_time32, clockid_t, which_clock, int, flags,
                struct old_timespec32 __user *, rqtp,
                struct old_timespec32 __user *, rmtp)
{
        const struct k_clock *kc = clockid_to_kclock(which_clock);
        struct timespec64 t;

        if (!kc)
                return -EINVAL;
        if (!kc->nsleep)
                return -EOPNOTSUPP;

        if (get_old_timespec32(&t, rqtp))
                return -EFAULT;

        if (!timespec64_valid(&t))
                return -EINVAL;
        if (flags & TIMER_ABSTIME)
                rmtp = NULL;
        current->restart_block.fn = do_no_restart_syscall;
        current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
        current->restart_block.nanosleep.compat_rmtp = rmtp;

        return kc->nsleep(which_clock, flags, &t);
}

#endif

static const struct k_clock clock_realtime = {
        .clock_getres                = posix_get_hrtimer_res,
        .clock_get_timespec        = posix_get_realtime_timespec,
        .clock_get_ktime        = posix_get_realtime_ktime,
        .clock_set                = posix_clock_realtime_set,
        .clock_adj                = posix_clock_realtime_adj,
        .nsleep                        = common_nsleep,
        .timer_create                = common_timer_create,
        .timer_set                = common_timer_set,
        .timer_get                = common_timer_get,
        .timer_del                = common_timer_del,
        .timer_rearm                = common_hrtimer_rearm,
        .timer_forward                = common_hrtimer_forward,
        .timer_remaining        = common_hrtimer_remaining,
        .timer_try_to_cancel        = common_hrtimer_try_to_cancel,
        .timer_wait_running        = common_timer_wait_running,
        .timer_arm                = common_hrtimer_arm,
};

static const struct k_clock clock_monotonic = {
        .clock_getres                = posix_get_hrtimer_res,
        .clock_get_timespec        = posix_get_monotonic_timespec,
        .clock_get_ktime        = posix_get_monotonic_ktime,
        .nsleep                        = common_nsleep_timens,
        .timer_create                = common_timer_create,
        .timer_set                = common_timer_set,
        .timer_get                = common_timer_get,
        .timer_del                = common_timer_del,
        .timer_rearm                = common_hrtimer_rearm,
        .timer_forward                = common_hrtimer_forward,
        .timer_remaining        = common_hrtimer_remaining,
        .timer_try_to_cancel        = common_hrtimer_try_to_cancel,
        .timer_wait_running        = common_timer_wait_running,
        .timer_arm                = common_hrtimer_arm,
};

static const struct k_clock clock_monotonic_raw = {
        .clock_getres                = posix_get_hrtimer_res,
        .clock_get_timespec        = posix_get_monotonic_raw,
};

static const struct k_clock clock_realtime_coarse = {
        .clock_getres                = posix_get_coarse_res,
        .clock_get_timespec        = posix_get_realtime_coarse,
};

static const struct k_clock clock_monotonic_coarse = {
        .clock_getres                = posix_get_coarse_res,
        .clock_get_timespec        = posix_get_monotonic_coarse,
};

static const struct k_clock clock_tai = {
        .clock_getres                = posix_get_hrtimer_res,
        .clock_get_ktime        = posix_get_tai_ktime,
        .clock_get_timespec        = posix_get_tai_timespec,
        .nsleep                        = common_nsleep,
        .timer_create                = common_timer_create,
        .timer_set                = common_timer_set,
        .timer_get                = common_timer_get,
        .timer_del                = common_timer_del,
        .timer_rearm                = common_hrtimer_rearm,
        .timer_forward                = common_hrtimer_forward,
        .timer_remaining        = common_hrtimer_remaining,
        .timer_try_to_cancel        = common_hrtimer_try_to_cancel,
        .timer_wait_running        = common_timer_wait_running,
        .timer_arm                = common_hrtimer_arm,
};

static const struct k_clock clock_boottime = {
        .clock_getres                = posix_get_hrtimer_res,
        .clock_get_ktime        = posix_get_boottime_ktime,
        .clock_get_timespec        = posix_get_boottime_timespec,
        .nsleep                        = common_nsleep_timens,
        .timer_create                = common_timer_create,
        .timer_set                = common_timer_set,
        .timer_get                = common_timer_get,
        .timer_del                = common_timer_del,
        .timer_rearm                = common_hrtimer_rearm,
        .timer_forward                = common_hrtimer_forward,
        .timer_remaining        = common_hrtimer_remaining,
        .timer_try_to_cancel        = common_hrtimer_try_to_cancel,
        .timer_wait_running        = common_timer_wait_running,
        .timer_arm                = common_hrtimer_arm,
};

static const struct k_clock * const posix_clocks[] = {
        [CLOCK_REALTIME]                = &clock_realtime,
        [CLOCK_MONOTONIC]                = &clock_monotonic,
        [CLOCK_PROCESS_CPUTIME_ID]        = &clock_process,
        [CLOCK_THREAD_CPUTIME_ID]        = &clock_thread,
        [CLOCK_MONOTONIC_RAW]                = &clock_monotonic_raw,
        [CLOCK_REALTIME_COARSE]                = &clock_realtime_coarse,
        [CLOCK_MONOTONIC_COARSE]        = &clock_monotonic_coarse,
        [CLOCK_BOOTTIME]                = &clock_boottime,
        [CLOCK_REALTIME_ALARM]                = &alarm_clock,
        [CLOCK_BOOTTIME_ALARM]                = &alarm_clock,
        [CLOCK_TAI]                        = &clock_tai,
#ifdef CONFIG_POSIX_AUX_CLOCKS
        [CLOCK_AUX ... CLOCK_AUX_LAST]        = &clock_aux,
#endif
};

static const struct k_clock *clockid_to_kclock(const clockid_t id)
{
        clockid_t idx = id;

        if (id < 0) {
                return (id & CLOCKFD_MASK) == CLOCKFD ?
                        &clock_posix_dynamic : &clock_posix_cpu;
        }

        if (id >= ARRAY_SIZE(posix_clocks))
                return NULL;

        return posix_clocks[array_index_nospec(idx, ARRAY_SIZE(posix_clocks))];
}

static int __init posixtimer_init(void)
{
        unsigned long i, size;
        unsigned int shift;

        posix_timers_cache = kmem_cache_create("posix_timers_cache",
                                               sizeof(struct k_itimer),
                                               __alignof__(struct k_itimer),
                                               SLAB_ACCOUNT, NULL);

        if (IS_ENABLED(CONFIG_BASE_SMALL))
                size = 512;
        else
                size = roundup_pow_of_two(512 * num_possible_cpus());

        timer_buckets = alloc_large_system_hash("posixtimers", sizeof(*timer_buckets),
                                                size, 0, 0, &shift, NULL, size, size);
        size = 1UL << shift;
        timer_hashmask = size - 1;

        for (i = 0; i < size; i++) {
                spin_lock_init(&timer_buckets[i].lock);
                INIT_HLIST_HEAD(&timer_buckets[i].head);
        }
        return 0;
}
core_initcall(posixtimer_init);







































































































   32 
    2 

    8 










   34 






   34 






   34 




    9 












    9 
    9 


    9 
    9 
    9 









    4 














    4 
    4 



    4 
    4 
    4 









   13 


    9 

    4 






   13 


    9 

    4 




   37 



   13 

   34 




   37 

   13 

   34 






   36 






   36 










   36 
























    2 






























    2 












    2 
    2 













    2 










    2 























































































































   15 



   15 






























































































    6 

















   26 







   58 




   23 

   43 















    4 


   22 














    4 






    4 













    4 



    4 











































































































































































































   12 









    8 

    4 










   12 



   12 





   12 


   12 












   30 





















   12 


   12 

   12 





   25 


    8 





   15 






    7 







   13 






   11 






   22 






















   12 








   12 



















   24 







   14 







   12 







   22 
    2 







   29 





   12 



   24 





   29 
   29 
















   29 

















































































































































   17 


    1 



   17 










    3 


    1 



    3 












   38 




    2 
    2 








   12 

    1 






    1 


   35 

    1 







   33 










    6 




















   14 



   21 

   19 


   17 


    3 


   11 
    7 



























































































































    1 


    1 
















    2 

















    2 

    2 










    2 








    2 









    2 










    2 
    2 




    1 






    2 













    5 



















    1 





    1 













    1 























    1 





    1 













































    4 
    2 








    4 














    4 












    2 








































    4 















    4 








    2 












    4 


    4 


    4 
    4 







    4 











    2 


    2 

    4 



    4 

    4 






















    4 




































    4 


    4 











    2 
    4 
    4 














    2 

    2 



    2 



    2 






    2 





    2 






    2 




    2 
    2 
    2 










    5 


    5 




















    5 

    5 




    5 











    2 




    2 



    2 

    2 

    2 




    2 






    2 



    2 
    2 
    2 




    2 






    2 
















   22 


    1 
    5 






    5 
    2 








    7 



    7 
    7 




    7 


    3 
    4 












   26 






   26 



















   26 
   26 


    7 



   24 
    9 
   17 

   24 















    2 
    3 




   15 







    4 








    5 




    5 





    5 





    5 




































































































































































    1 
    1 



    1 

























    5 
    1 
    1 













    5 
    5 


























    5 





























    5 


    5 
















    5 















    5 


















    5 




    5 








    1 









    1 
    1 






    5 




    5 























   23 





    9 

   19 

    5 
    5 












    2 




   16 
   11 


    5 
    6 





    2 




    4 
























    5 








  939 







  938 
    2 


   11 
   14 


   11 


   20 
    3 



















































    3 



   11 
   11 




   11 
    7 
    7 

   11 
   11 
   11 




   11 
   11 

   11 
   11 


   11 
    5 




    6 











   11 

    6 
    5 

   10 
    1 

   11 


































   15 








    5 





    8 



    5 






















   12 








    4 

   11 





























   18 
   11 



   22 
    7 





   15 


   12 



   24 
    9 





   11 
   15 













   14 
   22 
   22 






   11 

   27 

    9 







   30 

    8 
    8 













    8 
    6 






    5 




    7 






   18 



    3 



   15 



    6 



    7 
    1 




   13 







   13 















    2 
    2 



    1 



   53 




   42 



    3 













   68 
   21 


    4 
    3 
    7 



    1 





   77 

    1 








    1 





   15 


   23 
    1 





   11 


    9 
   18 


   26 
    1 






    1 
    2 
    3 



    7 




   34 
    1 










    1 


    6 



    1 


    2 








   23 
    1 









    1 





    9 


   20 
    1 



   26 
    1 













   22 

   16 

    2 
   12 


    2 




    7 



   11 











  101 






    1 


    9 


   91 
   28 


   45 
    1 






   45 



   33 
   33 














   12 
   12 




    3 





    3 




    2 







   17 











    1 


    4 


    6 





    6 

    4 











   13 








    1 


    4 


    8 




    2 



































    5 



    5 


    5 


    1 


    4 
    4 



    3 









    1 













   21 


    2 


    9 

   15 


    2 


   13 





   16 
   16 













    1 


    6 


   17 

   17 


























































   10 


    6 





    5 













    4 









   24 






   24 



    7 



    7 






    2 



    2 




    1 

















   14 

    6 
    2 





















    7 


   20 



   11 






    3 


    5 






   15 







































































    1 


    1 
    1 








   27 






   27 







    2 




    3 




    3 
















    1 














    1 


    6 


    6 




    1 



    1 










   15 









   15 










































  303 

    8 
    7 








 2703 



 2546 





  803 
  186 



  303 


















   29 













   29 



   29 











    4 





    4 





    4 










   27 


   27 

   18 







   27 


   27 





    4 





    4 

    4 



    4 






    4 
    4 




    4 


    4 




















































   25 






   25 


















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
// SPDX-License-Identifier: GPL-2.0
/* Generic nexthop implementation
 *
 * Copyright (c) 2017-19 Cumulus Networks
 * Copyright (c) 2017-19 David Ahern <dsa@cumulusnetworks.com>
 */

#include <linux/nexthop.h>
#include <linux/rtnetlink.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <net/arp.h>
#include <net/ip6_route.h>
#include <net/lwtunnel.h>
#include <net/ndisc.h>
#include <net/nexthop.h>
#include <net/route.h>
#include <net/sock.h>

#define NH_RES_DEFAULT_IDLE_TIMER        (120 * HZ)
#define NH_RES_DEFAULT_UNBALANCED_TIMER        0        /* No forced rebalancing. */

static bool __must_check remove_nexthop(struct net *net, struct nexthop *nh,
                                        struct nl_info *nlinfo);

#define NH_DEV_HASHBITS  8
#define NH_DEV_HASHSIZE (1U << NH_DEV_HASHBITS)

#define NHA_OP_FLAGS_DUMP_ALL (NHA_OP_FLAG_DUMP_STATS |                \
                               NHA_OP_FLAG_DUMP_HW_STATS)

static const struct nla_policy rtm_nh_policy_new[] = {
        [NHA_ID]                = { .type = NLA_U32 },
        [NHA_GROUP]                = { .type = NLA_BINARY },
        [NHA_GROUP_TYPE]        = { .type = NLA_U16 },
        [NHA_BLACKHOLE]                = { .type = NLA_FLAG },
        [NHA_OIF]                = { .type = NLA_U32 },
        [NHA_GATEWAY]                = { .type = NLA_BINARY },
        [NHA_ENCAP_TYPE]        = { .type = NLA_U16 },
        [NHA_ENCAP]                = { .type = NLA_NESTED },
        [NHA_FDB]                = { .type = NLA_FLAG },
        [NHA_DST_PORT]                = NLA_POLICY_MIN(NLA_BE16, 1),
        [NHA_RES_GROUP]                = { .type = NLA_NESTED },
        [NHA_HW_STATS_ENABLE]        = NLA_POLICY_MAX(NLA_U32, true),
};

static const struct nla_policy rtm_nh_policy_get[] = {
        [NHA_ID]                = { .type = NLA_U32 },
        [NHA_OP_FLAGS]                = NLA_POLICY_MASK(NLA_U32,
                                                  NHA_OP_FLAGS_DUMP_ALL),
};

static const struct nla_policy rtm_nh_policy_del[] = {
        [NHA_ID]                = { .type = NLA_U32 },
};

static const struct nla_policy rtm_nh_policy_dump[] = {
        [NHA_OIF]                = { .type = NLA_U32 },
        [NHA_GROUPS]                = { .type = NLA_FLAG },
        [NHA_MASTER]                = { .type = NLA_U32 },
        [NHA_FDB]                = { .type = NLA_FLAG },
        [NHA_OP_FLAGS]                = NLA_POLICY_MASK(NLA_U32,
                                                  NHA_OP_FLAGS_DUMP_ALL),
};

static const struct nla_policy rtm_nh_res_policy_new[] = {
        [NHA_RES_GROUP_BUCKETS]                        = { .type = NLA_U16 },
        [NHA_RES_GROUP_IDLE_TIMER]                = { .type = NLA_U32 },
        [NHA_RES_GROUP_UNBALANCED_TIMER]        = { .type = NLA_U32 },
};

static const struct nla_policy rtm_nh_policy_dump_bucket[] = {
        [NHA_ID]                = { .type = NLA_U32 },
        [NHA_OIF]                = { .type = NLA_U32 },
        [NHA_MASTER]                = { .type = NLA_U32 },
        [NHA_RES_BUCKET]        = { .type = NLA_NESTED },
};

static const struct nla_policy rtm_nh_res_bucket_policy_dump[] = {
        [NHA_RES_BUCKET_NH_ID]        = { .type = NLA_U32 },
};

static const struct nla_policy rtm_nh_policy_get_bucket[] = {
        [NHA_ID]                = { .type = NLA_U32 },
        [NHA_RES_BUCKET]        = { .type = NLA_NESTED },
};

static const struct nla_policy rtm_nh_res_bucket_policy_get[] = {
        [NHA_RES_BUCKET_INDEX]        = { .type = NLA_U16 },
};

static bool nexthop_notifiers_is_empty(struct net *net)
{
        return !net->nexthop.notifier_chain.head;
}

static void
__nh_notifier_single_info_init(struct nh_notifier_single_info *nh_info,
                               const struct nh_info *nhi)
{
        nh_info->dev = nhi->fib_nhc.nhc_dev;
        nh_info->gw_family = nhi->fib_nhc.nhc_gw_family;
        if (nh_info->gw_family == AF_INET)
                nh_info->ipv4 = nhi->fib_nhc.nhc_gw.ipv4;
        else if (nh_info->gw_family == AF_INET6)
                nh_info->ipv6 = nhi->fib_nhc.nhc_gw.ipv6;

        nh_info->id = nhi->nh_parent->id;
        nh_info->is_reject = nhi->reject_nh;
        nh_info->is_fdb = nhi->fdb_nh;
        nh_info->has_encap = !!nhi->fib_nhc.nhc_lwtstate;
}

static int nh_notifier_single_info_init(struct nh_notifier_info *info,
                                        const struct nexthop *nh)
{
        struct nh_info *nhi = rtnl_dereference(nh->nh_info);

        info->type = NH_NOTIFIER_INFO_TYPE_SINGLE;
        info->nh = kzalloc_obj(*info->nh);
        if (!info->nh)
                return -ENOMEM;

        __nh_notifier_single_info_init(info->nh, nhi);

        return 0;
}

static void nh_notifier_single_info_fini(struct nh_notifier_info *info)
{
        kfree(info->nh);
}

static int nh_notifier_mpath_info_init(struct nh_notifier_info *info,
                                       struct nh_group *nhg)
{
        u16 num_nh = nhg->num_nh;
        int i;

        info->type = NH_NOTIFIER_INFO_TYPE_GRP;
        info->nh_grp = kzalloc_flex(*info->nh_grp, nh_entries, num_nh);
        if (!info->nh_grp)
                return -ENOMEM;

        info->nh_grp->num_nh = num_nh;
        info->nh_grp->is_fdb = nhg->fdb_nh;
        info->nh_grp->hw_stats = nhg->hw_stats;

        for (i = 0; i < num_nh; i++) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];
                struct nh_info *nhi;

                nhi = rtnl_dereference(nhge->nh->nh_info);
                info->nh_grp->nh_entries[i].weight = nhge->weight;
                __nh_notifier_single_info_init(&info->nh_grp->nh_entries[i].nh,
                                               nhi);
        }

        return 0;
}

static int nh_notifier_res_table_info_init(struct nh_notifier_info *info,
                                           struct nh_group *nhg)
{
        struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);
        u16 num_nh_buckets = res_table->num_nh_buckets;
        unsigned long size;
        u16 i;

        info->type = NH_NOTIFIER_INFO_TYPE_RES_TABLE;
        size = struct_size(info->nh_res_table, nhs, num_nh_buckets);
        info->nh_res_table = __vmalloc(size, GFP_KERNEL | __GFP_ZERO |
                                       __GFP_NOWARN);
        if (!info->nh_res_table)
                return -ENOMEM;

        info->nh_res_table->num_nh_buckets = num_nh_buckets;
        info->nh_res_table->hw_stats = nhg->hw_stats;

        for (i = 0; i < num_nh_buckets; i++) {
                struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
                struct nh_grp_entry *nhge;
                struct nh_info *nhi;

                nhge = rtnl_dereference(bucket->nh_entry);
                nhi = rtnl_dereference(nhge->nh->nh_info);
                __nh_notifier_single_info_init(&info->nh_res_table->nhs[i],
                                               nhi);
        }

        return 0;
}

static int nh_notifier_grp_info_init(struct nh_notifier_info *info,
                                     const struct nexthop *nh)
{
        struct nh_group *nhg = rtnl_dereference(nh->nh_grp);

        if (nhg->hash_threshold)
                return nh_notifier_mpath_info_init(info, nhg);
        else if (nhg->resilient)
                return nh_notifier_res_table_info_init(info, nhg);
        return -EINVAL;
}

static void nh_notifier_grp_info_fini(struct nh_notifier_info *info,
                                      const struct nexthop *nh)
{
        struct nh_group *nhg = rtnl_dereference(nh->nh_grp);

        if (nhg->hash_threshold)
                kfree(info->nh_grp);
        else if (nhg->resilient)
                vfree(info->nh_res_table);
}

static int nh_notifier_info_init(struct nh_notifier_info *info,
                                 const struct nexthop *nh)
{
        info->id = nh->id;

        if (nh->is_group)
                return nh_notifier_grp_info_init(info, nh);
        else
                return nh_notifier_single_info_init(info, nh);
}

static void nh_notifier_info_fini(struct nh_notifier_info *info,
                                  const struct nexthop *nh)
{
        if (nh->is_group)
                nh_notifier_grp_info_fini(info, nh);
        else
                nh_notifier_single_info_fini(info);
}

static int call_nexthop_notifiers(struct net *net,
                                  enum nexthop_event_type event_type,
                                  struct nexthop *nh,
                                  struct netlink_ext_ack *extack)
{
        struct nh_notifier_info info = {
                .net = net,
                .extack = extack,
        };
        int err;

        ASSERT_RTNL();

        if (nexthop_notifiers_is_empty(net))
                return 0;

        err = nh_notifier_info_init(&info, nh);
        if (err) {
                NL_SET_ERR_MSG(extack, "Failed to initialize nexthop notifier info");
                return err;
        }

        err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
                                           event_type, &info);
        nh_notifier_info_fini(&info, nh);

        return notifier_to_errno(err);
}

static int
nh_notifier_res_bucket_idle_timer_get(const struct nh_notifier_info *info,
                                      bool force, unsigned int *p_idle_timer_ms)
{
        struct nh_res_table *res_table;
        struct nh_group *nhg;
        struct nexthop *nh;
        int err = 0;

        /* When 'force' is false, nexthop bucket replacement is performed
         * because the bucket was deemed to be idle. In this case, capable
         * listeners can choose to perform an atomic replacement: The bucket is
         * only replaced if it is inactive. However, if the idle timer interval
         * is smaller than the interval in which a listener is querying
         * buckets' activity from the device, then atomic replacement should
         * not be tried. Pass the idle timer value to listeners, so that they
         * could determine which type of replacement to perform.
         */
        if (force) {
                *p_idle_timer_ms = 0;
                return 0;
        }

        rcu_read_lock();

        nh = nexthop_find_by_id(info->net, info->id);
        if (!nh) {
                err = -EINVAL;
                goto out;
        }

        nhg = rcu_dereference(nh->nh_grp);
        res_table = rcu_dereference(nhg->res_table);
        *p_idle_timer_ms = jiffies_to_msecs(res_table->idle_timer);

out:
        rcu_read_unlock();

        return err;
}

static int nh_notifier_res_bucket_info_init(struct nh_notifier_info *info,
                                            u16 bucket_index, bool force,
                                            struct nh_info *oldi,
                                            struct nh_info *newi)
{
        unsigned int idle_timer_ms;
        int err;

        err = nh_notifier_res_bucket_idle_timer_get(info, force,
                                                    &idle_timer_ms);
        if (err)
                return err;

        info->type = NH_NOTIFIER_INFO_TYPE_RES_BUCKET;
        info->nh_res_bucket = kzalloc_obj(*info->nh_res_bucket);
        if (!info->nh_res_bucket)
                return -ENOMEM;

        info->nh_res_bucket->bucket_index = bucket_index;
        info->nh_res_bucket->idle_timer_ms = idle_timer_ms;
        info->nh_res_bucket->force = force;
        __nh_notifier_single_info_init(&info->nh_res_bucket->old_nh, oldi);
        __nh_notifier_single_info_init(&info->nh_res_bucket->new_nh, newi);
        return 0;
}

static void nh_notifier_res_bucket_info_fini(struct nh_notifier_info *info)
{
        kfree(info->nh_res_bucket);
}

static int __call_nexthop_res_bucket_notifiers(struct net *net, u32 nhg_id,
                                               u16 bucket_index, bool force,
                                               struct nh_info *oldi,
                                               struct nh_info *newi,
                                               struct netlink_ext_ack *extack)
{
        struct nh_notifier_info info = {
                .net = net,
                .extack = extack,
                .id = nhg_id,
        };
        int err;

        if (nexthop_notifiers_is_empty(net))
                return 0;

        err = nh_notifier_res_bucket_info_init(&info, bucket_index, force,
                                               oldi, newi);
        if (err)
                return err;

        err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
                                           NEXTHOP_EVENT_BUCKET_REPLACE, &info);
        nh_notifier_res_bucket_info_fini(&info);

        return notifier_to_errno(err);
}

/* There are three users of RES_TABLE, and NHs etc. referenced from there:
 *
 * 1) a collection of callbacks for NH maintenance. This operates under
 *    RTNL,
 * 2) the delayed work that gradually balances the resilient table,
 * 3) and nexthop_select_path(), operating under RCU.
 *
 * Both the delayed work and the RTNL block are writers, and need to
 * maintain mutual exclusion. Since there are only two and well-known
 * writers for each table, the RTNL code can make sure it has exclusive
 * access thus:
 *
 * - Have the DW operate without locking;
 * - synchronously cancel the DW;
 * - do the writing;
 * - if the write was not actually a delete, call upkeep, which schedules
 *   DW again if necessary.
 *
 * The functions that are always called from the RTNL context use
 * rtnl_dereference(). The functions that can also be called from the DW do
 * a raw dereference and rely on the above mutual exclusion scheme.
 */
#define nh_res_dereference(p) (rcu_dereference_raw(p))

static int call_nexthop_res_bucket_notifiers(struct net *net, u32 nhg_id,
                                             u16 bucket_index, bool force,
                                             struct nexthop *old_nh,
                                             struct nexthop *new_nh,
                                             struct netlink_ext_ack *extack)
{
        struct nh_info *oldi = nh_res_dereference(old_nh->nh_info);
        struct nh_info *newi = nh_res_dereference(new_nh->nh_info);

        return __call_nexthop_res_bucket_notifiers(net, nhg_id, bucket_index,
                                                   force, oldi, newi, extack);
}

static int call_nexthop_res_table_notifiers(struct net *net, struct nexthop *nh,
                                            struct netlink_ext_ack *extack)
{
        struct nh_notifier_info info = {
                .net = net,
                .extack = extack,
                .id = nh->id,
        };
        struct nh_group *nhg;
        int err;

        ASSERT_RTNL();

        if (nexthop_notifiers_is_empty(net))
                return 0;

        /* At this point, the nexthop buckets are still not populated. Only
         * emit a notification with the logical nexthops, so that a listener
         * could potentially veto it in case of unsupported configuration.
         */
        nhg = rtnl_dereference(nh->nh_grp);
        err = nh_notifier_mpath_info_init(&info, nhg);
        if (err) {
                NL_SET_ERR_MSG(extack, "Failed to initialize nexthop notifier info");
                return err;
        }

        err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
                                           NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE,
                                           &info);
        kfree(info.nh_grp);

        return notifier_to_errno(err);
}

static int call_nexthop_notifier(struct notifier_block *nb, struct net *net,
                                 enum nexthop_event_type event_type,
                                 struct nexthop *nh,
                                 struct netlink_ext_ack *extack)
{
        struct nh_notifier_info info = {
                .net = net,
                .extack = extack,
        };
        int err;

        err = nh_notifier_info_init(&info, nh);
        if (err)
                return err;

        err = nb->notifier_call(nb, event_type, &info);
        nh_notifier_info_fini(&info, nh);

        return notifier_to_errno(err);
}

static unsigned int nh_dev_hashfn(unsigned int val)
{
        unsigned int mask = NH_DEV_HASHSIZE - 1;

        return (val ^
                (val >> NH_DEV_HASHBITS) ^
                (val >> (NH_DEV_HASHBITS * 2))) & mask;
}

static void nexthop_devhash_add(struct net *net, struct nh_info *nhi)
{
        struct net_device *dev = nhi->fib_nhc.nhc_dev;
        struct hlist_head *head;
        unsigned int hash;

        WARN_ON(!dev);

        hash = nh_dev_hashfn(dev->ifindex);
        head = &net->nexthop.devhash[hash];
        hlist_add_head(&nhi->dev_hash, head);
}

static void nexthop_free_group(struct nexthop *nh)
{
        struct nh_group *nhg;
        int i;

        nhg = rcu_dereference_raw(nh->nh_grp);
        for (i = 0; i < nhg->num_nh; ++i) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                WARN_ON(!list_empty(&nhge->nh_list));
                free_percpu(nhge->stats);
                nexthop_put(nhge->nh);
        }

        WARN_ON(nhg->spare == nhg);

        if (nhg->resilient)
                vfree(rcu_dereference_raw(nhg->res_table));

        kfree(nhg->spare);
        kfree(nhg);
}

static void nexthop_free_single(struct nexthop *nh)
{
        struct nh_info *nhi;

        nhi = rcu_dereference_raw(nh->nh_info);
        switch (nhi->family) {
        case AF_INET:
                fib_nh_release(nh->net, &nhi->fib_nh);
                break;
        case AF_INET6:
                fib6_nh_release(&nhi->fib6_nh);
                break;
        }
        kfree(nhi);
}

void nexthop_free_rcu(struct rcu_head *head)
{
        struct nexthop *nh = container_of(head, struct nexthop, rcu);

        if (nh->is_group)
                nexthop_free_group(nh);
        else
                nexthop_free_single(nh);

        kfree(nh);
}
EXPORT_SYMBOL_GPL(nexthop_free_rcu);

static struct nexthop *nexthop_alloc(void)
{
        struct nexthop *nh;

        nh = kzalloc_obj(struct nexthop);
        if (nh) {
                INIT_LIST_HEAD(&nh->fi_list);
                INIT_LIST_HEAD(&nh->f6i_list);
                INIT_LIST_HEAD(&nh->grp_list);
                INIT_LIST_HEAD(&nh->fdb_list);
                spin_lock_init(&nh->lock);
        }
        return nh;
}

static struct nh_group *nexthop_grp_alloc(u16 num_nh)
{
        struct nh_group *nhg;

        nhg = kzalloc_flex(*nhg, nh_entries, num_nh);
        if (nhg)
                nhg->num_nh = num_nh;

        return nhg;
}

static void nh_res_table_upkeep_dw(struct work_struct *work);

static struct nh_res_table *
nexthop_res_table_alloc(struct net *net, u32 nhg_id, struct nh_config *cfg)
{
        const u16 num_nh_buckets = cfg->nh_grp_res_num_buckets;
        struct nh_res_table *res_table;
        unsigned long size;

        size = struct_size(res_table, nh_buckets, num_nh_buckets);
        res_table = __vmalloc(size, GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN);
        if (!res_table)
                return NULL;

        res_table->net = net;
        res_table->nhg_id = nhg_id;
        INIT_DELAYED_WORK(&res_table->upkeep_dw, &nh_res_table_upkeep_dw);
        INIT_LIST_HEAD(&res_table->uw_nh_entries);
        res_table->idle_timer = cfg->nh_grp_res_idle_timer;
        res_table->unbalanced_timer = cfg->nh_grp_res_unbalanced_timer;
        res_table->num_nh_buckets = num_nh_buckets;
        return res_table;
}

static void nh_base_seq_inc(struct net *net)
{
        while (++net->nexthop.seq == 0)
                ;
}

/* no reference taken; rcu lock or rtnl must be held */
struct nexthop *nexthop_find_by_id(struct net *net, u32 id)
{
        struct rb_node **pp, *parent = NULL, *next;

        pp = &net->nexthop.rb_root.rb_node;
        while (1) {
                struct nexthop *nh;

                next = rcu_dereference_raw(*pp);
                if (!next)
                        break;
                parent = next;

                nh = rb_entry(parent, struct nexthop, rb_node);
                if (id < nh->id)
                        pp = &next->rb_left;
                else if (id > nh->id)
                        pp = &next->rb_right;
                else
                        return nh;
        }
        return NULL;
}
EXPORT_SYMBOL_GPL(nexthop_find_by_id);

/* used for auto id allocation; called with rtnl held */
static u32 nh_find_unused_id(struct net *net)
{
        u32 id_start = net->nexthop.last_id_allocated;

        while (1) {
                net->nexthop.last_id_allocated++;
                if (net->nexthop.last_id_allocated == id_start)
                        break;

                if (!nexthop_find_by_id(net, net->nexthop.last_id_allocated))
                        return net->nexthop.last_id_allocated;
        }
        return 0;
}

static void nh_res_time_set_deadline(unsigned long next_time,
                                     unsigned long *deadline)
{
        if (time_before(next_time, *deadline))
                *deadline = next_time;
}

static clock_t nh_res_table_unbalanced_time(struct nh_res_table *res_table)
{
        if (list_empty(&res_table->uw_nh_entries))
                return 0;
        return jiffies_delta_to_clock_t(jiffies - res_table->unbalanced_since);
}

static int nla_put_nh_group_res(struct sk_buff *skb, struct nh_group *nhg)
{
        struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);
        struct nlattr *nest;

        nest = nla_nest_start(skb, NHA_RES_GROUP);
        if (!nest)
                return -EMSGSIZE;

        if (nla_put_u16(skb, NHA_RES_GROUP_BUCKETS,
                        res_table->num_nh_buckets) ||
            nla_put_u32(skb, NHA_RES_GROUP_IDLE_TIMER,
                        jiffies_to_clock_t(res_table->idle_timer)) ||
            nla_put_u32(skb, NHA_RES_GROUP_UNBALANCED_TIMER,
                        jiffies_to_clock_t(res_table->unbalanced_timer)) ||
            nla_put_u64_64bit(skb, NHA_RES_GROUP_UNBALANCED_TIME,
                              nh_res_table_unbalanced_time(res_table),
                              NHA_RES_GROUP_PAD))
                goto nla_put_failure;

        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static void nh_grp_entry_stats_inc(struct nh_grp_entry *nhge)
{
        struct nh_grp_entry_stats *cpu_stats;

        cpu_stats = get_cpu_ptr(nhge->stats);
        u64_stats_update_begin(&cpu_stats->syncp);
        u64_stats_inc(&cpu_stats->packets);
        u64_stats_update_end(&cpu_stats->syncp);
        put_cpu_ptr(cpu_stats);
}

static void nh_grp_entry_stats_read(struct nh_grp_entry *nhge,
                                    u64 *ret_packets)
{
        int i;

        *ret_packets = 0;

        for_each_possible_cpu(i) {
                struct nh_grp_entry_stats *cpu_stats;
                unsigned int start;
                u64 packets;

                cpu_stats = per_cpu_ptr(nhge->stats, i);
                do {
                        start = u64_stats_fetch_begin(&cpu_stats->syncp);
                        packets = u64_stats_read(&cpu_stats->packets);
                } while (u64_stats_fetch_retry(&cpu_stats->syncp, start));

                *ret_packets += packets;
        }
}

static int nh_notifier_grp_hw_stats_init(struct nh_notifier_info *info,
                                         const struct nexthop *nh)
{
        struct nh_group *nhg;
        int i;

        ASSERT_RTNL();
        nhg = rtnl_dereference(nh->nh_grp);

        info->id = nh->id;
        info->type = NH_NOTIFIER_INFO_TYPE_GRP_HW_STATS;
        info->nh_grp_hw_stats = kzalloc_flex(*info->nh_grp_hw_stats, stats,
                                             nhg->num_nh);
        if (!info->nh_grp_hw_stats)
                return -ENOMEM;

        info->nh_grp_hw_stats->num_nh = nhg->num_nh;
        for (i = 0; i < nhg->num_nh; i++) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                info->nh_grp_hw_stats->stats[i].id = nhge->nh->id;
        }

        return 0;
}

static void nh_notifier_grp_hw_stats_fini(struct nh_notifier_info *info)
{
        kfree(info->nh_grp_hw_stats);
}

void nh_grp_hw_stats_report_delta(struct nh_notifier_grp_hw_stats_info *info,
                                  unsigned int nh_idx,
                                  u64 delta_packets)
{
        info->hw_stats_used = true;
        info->stats[nh_idx].packets += delta_packets;
}
EXPORT_SYMBOL(nh_grp_hw_stats_report_delta);

static void nh_grp_hw_stats_apply_update(struct nexthop *nh,
                                         struct nh_notifier_info *info)
{
        struct nh_group *nhg;
        int i;

        ASSERT_RTNL();
        nhg = rtnl_dereference(nh->nh_grp);

        for (i = 0; i < nhg->num_nh; i++) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                nhge->packets_hw += info->nh_grp_hw_stats->stats[i].packets;
        }
}

static int nh_grp_hw_stats_update(struct nexthop *nh, bool *hw_stats_used)
{
        struct nh_notifier_info info = {
                .net = nh->net,
        };
        struct net *net = nh->net;
        int err;

        if (nexthop_notifiers_is_empty(net)) {
                *hw_stats_used = false;
                return 0;
        }

        err = nh_notifier_grp_hw_stats_init(&info, nh);
        if (err)
                return err;

        err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
                                           NEXTHOP_EVENT_HW_STATS_REPORT_DELTA,
                                           &info);

        /* Cache whatever we got, even if there was an error, otherwise the
         * successful stats retrievals would get lost.
         */
        nh_grp_hw_stats_apply_update(nh, &info);
        *hw_stats_used = info.nh_grp_hw_stats->hw_stats_used;

        nh_notifier_grp_hw_stats_fini(&info);
        return notifier_to_errno(err);
}

static int nla_put_nh_group_stats_entry(struct sk_buff *skb,
                                        struct nh_grp_entry *nhge,
                                        u32 op_flags)
{
        struct nlattr *nest;
        u64 packets;

        nh_grp_entry_stats_read(nhge, &packets);

        nest = nla_nest_start(skb, NHA_GROUP_STATS_ENTRY);
        if (!nest)
                return -EMSGSIZE;

        if (nla_put_u32(skb, NHA_GROUP_STATS_ENTRY_ID, nhge->nh->id) ||
            nla_put_uint(skb, NHA_GROUP_STATS_ENTRY_PACKETS,
                         packets + nhge->packets_hw))
                goto nla_put_failure;

        if (op_flags & NHA_OP_FLAG_DUMP_HW_STATS &&
            nla_put_uint(skb, NHA_GROUP_STATS_ENTRY_PACKETS_HW,
                         nhge->packets_hw))
                goto nla_put_failure;

        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static int nla_put_nh_group_stats(struct sk_buff *skb, struct nexthop *nh,
                                  u32 op_flags)
{
        struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
        struct nlattr *nest;
        bool hw_stats_used;
        int err;
        int i;

        if (nla_put_u32(skb, NHA_HW_STATS_ENABLE, nhg->hw_stats))
                goto err_out;

        if (op_flags & NHA_OP_FLAG_DUMP_HW_STATS &&
            nhg->hw_stats) {
                err = nh_grp_hw_stats_update(nh, &hw_stats_used);
                if (err)
                        goto out;

                if (nla_put_u32(skb, NHA_HW_STATS_USED, hw_stats_used))
                        goto err_out;
        }

        nest = nla_nest_start(skb, NHA_GROUP_STATS);
        if (!nest)
                goto err_out;

        for (i = 0; i < nhg->num_nh; i++)
                if (nla_put_nh_group_stats_entry(skb, &nhg->nh_entries[i],
                                                 op_flags))
                        goto cancel_out;

        nla_nest_end(skb, nest);
        return 0;

cancel_out:
        nla_nest_cancel(skb, nest);
err_out:
        err = -EMSGSIZE;
out:
        return err;
}

static int nla_put_nh_group(struct sk_buff *skb, struct nexthop *nh,
                            u32 op_flags, u32 *resp_op_flags)
{
        struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
        struct nexthop_grp *p;
        size_t len = nhg->num_nh * sizeof(*p);
        struct nlattr *nla;
        u16 group_type = 0;
        u16 weight;
        int i;

        *resp_op_flags |= NHA_OP_FLAG_RESP_GRP_RESVD_0;

        if (nhg->hash_threshold)
                group_type = NEXTHOP_GRP_TYPE_MPATH;
        else if (nhg->resilient)
                group_type = NEXTHOP_GRP_TYPE_RES;

        if (nla_put_u16(skb, NHA_GROUP_TYPE, group_type))
                goto nla_put_failure;

        nla = nla_reserve(skb, NHA_GROUP, len);
        if (!nla)
                goto nla_put_failure;

        p = nla_data(nla);
        for (i = 0; i < nhg->num_nh; ++i) {
                weight = nhg->nh_entries[i].weight - 1;

                *p++ = (struct nexthop_grp) {
                        .id = nhg->nh_entries[i].nh->id,
                        .weight = weight,
                        .weight_high = weight >> 8,
                };
        }

        if (nhg->resilient && nla_put_nh_group_res(skb, nhg))
                goto nla_put_failure;

        if (op_flags & NHA_OP_FLAG_DUMP_STATS &&
            nla_put_nh_group_stats(skb, nh, op_flags))
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static int nh_fill_node(struct sk_buff *skb, struct nexthop *nh,
                        int event, u32 portid, u32 seq, unsigned int nlflags,
                        u32 op_flags)
{
        struct fib6_nh *fib6_nh;
        struct fib_nh *fib_nh;
        struct nlmsghdr *nlh;
        struct nh_info *nhi;
        struct nhmsg *nhm;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
        if (!nlh)
                return -EMSGSIZE;

        nhm = nlmsg_data(nlh);
        nhm->nh_family = AF_UNSPEC;
        nhm->nh_flags = nh->nh_flags;
        nhm->nh_protocol = nh->protocol;
        nhm->nh_scope = 0;
        nhm->resvd = 0;

        if (nla_put_u32(skb, NHA_ID, nh->id))
                goto nla_put_failure;

        if (nh->is_group) {
                struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
                u32 resp_op_flags = 0;

                if (nhg->fdb_nh && nla_put_flag(skb, NHA_FDB))
                        goto nla_put_failure;
                if (nla_put_nh_group(skb, nh, op_flags, &resp_op_flags) ||
                    nla_put_u32(skb, NHA_OP_FLAGS, resp_op_flags))
                        goto nla_put_failure;
                goto out;
        }

        nhi = rtnl_dereference(nh->nh_info);
        nhm->nh_family = nhi->family;
        if (nhi->reject_nh) {
                if (nla_put_flag(skb, NHA_BLACKHOLE))
                        goto nla_put_failure;
                goto out;
        } else if (nhi->fdb_nh) {
                if (nla_put_flag(skb, NHA_FDB))
                        goto nla_put_failure;
                if (nhi->dst_port &&
                    nla_put_be16(skb, NHA_DST_PORT, nhi->dst_port))
                        goto nla_put_failure;
        } else {
                const struct net_device *dev;

                dev = nhi->fib_nhc.nhc_dev;
                if (dev && nla_put_u32(skb, NHA_OIF, dev->ifindex))
                        goto nla_put_failure;
        }

        nhm->nh_scope = nhi->fib_nhc.nhc_scope;
        switch (nhi->family) {
        case AF_INET:
                fib_nh = &nhi->fib_nh;
                if (fib_nh->fib_nh_gw_family &&
                    nla_put_be32(skb, NHA_GATEWAY, fib_nh->fib_nh_gw4))
                        goto nla_put_failure;
                break;

        case AF_INET6:
                fib6_nh = &nhi->fib6_nh;
                if (fib6_nh->fib_nh_gw_family &&
                    nla_put_in6_addr(skb, NHA_GATEWAY, &fib6_nh->fib_nh_gw6))
                        goto nla_put_failure;
                break;
        }

        if (lwtunnel_fill_encap(skb, nhi->fib_nhc.nhc_lwtstate,
                                NHA_ENCAP, NHA_ENCAP_TYPE) < 0)
                goto nla_put_failure;

out:
        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static size_t nh_nlmsg_size_grp_res(struct nh_group *nhg)
{
        return nla_total_size(0) +        /* NHA_RES_GROUP */
                nla_total_size(2) +        /* NHA_RES_GROUP_BUCKETS */
                nla_total_size(4) +        /* NHA_RES_GROUP_IDLE_TIMER */
                nla_total_size(4) +        /* NHA_RES_GROUP_UNBALANCED_TIMER */
                nla_total_size_64bit(8);/* NHA_RES_GROUP_UNBALANCED_TIME */
}

static size_t nh_nlmsg_size_grp(struct nexthop *nh, u32 op_flags)
{
        struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
        size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh;
        size_t tot = nla_total_size(sz) +
                nla_total_size(2) +        /* NHA_GROUP_TYPE */
                nla_total_size(0);        /* NHA_FDB */

        if (nhg->resilient)
                tot += nh_nlmsg_size_grp_res(nhg);

        if (op_flags & NHA_OP_FLAG_DUMP_STATS) {
                tot += nla_total_size(0) +          /* NHA_GROUP_STATS */
                       nla_total_size(4);          /* NHA_HW_STATS_ENABLE */
                tot += nhg->num_nh *
                       (nla_total_size(0) +          /* NHA_GROUP_STATS_ENTRY */
                        nla_total_size(4) +          /* NHA_GROUP_STATS_ENTRY_ID */
                        nla_total_size_64bit(8)); /* NHA_GROUP_STATS_ENTRY_PACKETS */

                if (op_flags & NHA_OP_FLAG_DUMP_HW_STATS) {
                        tot += nhg->num_nh *
                               nla_total_size_64bit(8); /* NHA_GROUP_STATS_ENTRY_PACKETS_HW */
                        tot += nla_total_size(4);        /* NHA_HW_STATS_USED */
                }
        }

        return tot;
}

static size_t nh_nlmsg_size_single(struct nexthop *nh)
{
        struct nh_info *nhi = rtnl_dereference(nh->nh_info);
        size_t sz;

        /* covers NHA_BLACKHOLE since NHA_OIF and BLACKHOLE
         * are mutually exclusive
         */
        sz = nla_total_size(4);  /* NHA_OIF */

        switch (nhi->family) {
        case AF_INET:
                if (nhi->fib_nh.fib_nh_gw_family)
                        sz += nla_total_size(4);  /* NHA_GATEWAY */
                break;

        case AF_INET6:
                /* NHA_GATEWAY */
                if (nhi->fib6_nh.fib_nh_gw_family)
                        sz += nla_total_size(sizeof(const struct in6_addr));
                break;
        }

        if (nhi->dst_port)
                sz += nla_total_size(2);        /* NHA_DST_PORT */

        if (nhi->fib_nhc.nhc_lwtstate) {
                sz += lwtunnel_get_encap_size(nhi->fib_nhc.nhc_lwtstate);
                sz += nla_total_size(2);  /* NHA_ENCAP_TYPE */
        }

        return sz;
}

static size_t nh_nlmsg_size(struct nexthop *nh, u32 op_flags)
{
        size_t sz = NLMSG_ALIGN(sizeof(struct nhmsg));

        sz += nla_total_size(4); /* NHA_ID */

        if (nh->is_group)
                sz += nh_nlmsg_size_grp(nh, op_flags) +
                      nla_total_size(4) +        /* NHA_OP_FLAGS */
                      0;
        else
                sz += nh_nlmsg_size_single(nh);

        return sz;
}

static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info)
{
        unsigned int nlflags = info->nlh ? info->nlh->nlmsg_flags : 0;
        u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = nlmsg_new(nh_nlmsg_size(nh, 0), gfp_any());
        if (!skb)
                goto errout;

        err = nh_fill_node(skb, nh, event, info->portid, seq, nlflags, 0);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in nh_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_NEXTHOP,
                    info->nlh, gfp_any());
        return;
errout:
        rtnl_set_sk_err(info->nl_net, RTNLGRP_NEXTHOP, err);
}

static unsigned long nh_res_bucket_used_time(const struct nh_res_bucket *bucket)
{
        return (unsigned long)atomic_long_read(&bucket->used_time);
}

static unsigned long
nh_res_bucket_idle_point(const struct nh_res_table *res_table,
                         const struct nh_res_bucket *bucket,
                         unsigned long now)
{
        unsigned long time = nh_res_bucket_used_time(bucket);

        /* Bucket was not used since it was migrated. The idle time is now. */
        if (time == bucket->migrated_time)
                return now;

        return time + res_table->idle_timer;
}

static unsigned long
nh_res_table_unb_point(const struct nh_res_table *res_table)
{
        return res_table->unbalanced_since + res_table->unbalanced_timer;
}

static void nh_res_bucket_set_idle(const struct nh_res_table *res_table,
                                   struct nh_res_bucket *bucket)
{
        unsigned long now = jiffies;

        atomic_long_set(&bucket->used_time, (long)now);
        bucket->migrated_time = now;
}

static void nh_res_bucket_set_busy(struct nh_res_bucket *bucket)
{
        atomic_long_set(&bucket->used_time, (long)jiffies);
}

static clock_t nh_res_bucket_idle_time(const struct nh_res_bucket *bucket)
{
        unsigned long used_time = nh_res_bucket_used_time(bucket);

        return jiffies_delta_to_clock_t(jiffies - used_time);
}

static int nh_fill_res_bucket(struct sk_buff *skb, struct nexthop *nh,
                              struct nh_res_bucket *bucket, u16 bucket_index,
                              int event, u32 portid, u32 seq,
                              unsigned int nlflags,
                              struct netlink_ext_ack *extack)
{
        struct nh_grp_entry *nhge = nh_res_dereference(bucket->nh_entry);
        struct nlmsghdr *nlh;
        struct nlattr *nest;
        struct nhmsg *nhm;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
        if (!nlh)
                return -EMSGSIZE;

        nhm = nlmsg_data(nlh);
        nhm->nh_family = AF_UNSPEC;
        nhm->nh_flags = bucket->nh_flags;
        nhm->nh_protocol = nh->protocol;
        nhm->nh_scope = 0;
        nhm->resvd = 0;

        if (nla_put_u32(skb, NHA_ID, nh->id))
                goto nla_put_failure;

        nest = nla_nest_start(skb, NHA_RES_BUCKET);
        if (!nest)
                goto nla_put_failure;

        if (nla_put_u16(skb, NHA_RES_BUCKET_INDEX, bucket_index) ||
            nla_put_u32(skb, NHA_RES_BUCKET_NH_ID, nhge->nh->id) ||
            nla_put_u64_64bit(skb, NHA_RES_BUCKET_IDLE_TIME,
                              nh_res_bucket_idle_time(bucket),
                              NHA_RES_BUCKET_PAD))
                goto nla_put_failure_nest;

        nla_nest_end(skb, nest);
        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure_nest:
        nla_nest_cancel(skb, nest);
nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static void nexthop_bucket_notify(struct nh_res_table *res_table,
                                  u16 bucket_index)
{
        struct nh_res_bucket *bucket = &res_table->nh_buckets[bucket_index];
        struct nh_grp_entry *nhge = nh_res_dereference(bucket->nh_entry);
        struct nexthop *nh = nhge->nh_parent;
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb)
                goto errout;

        err = nh_fill_res_bucket(skb, nh, bucket, bucket_index,
                                 RTM_NEWNEXTHOPBUCKET, 0, 0, NLM_F_REPLACE,
                                 NULL);
        if (err < 0) {
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, nh->net, 0, RTNLGRP_NEXTHOP, NULL, GFP_KERNEL);
        return;
errout:
        rtnl_set_sk_err(nh->net, RTNLGRP_NEXTHOP, err);
}

static bool valid_group_nh(struct nexthop *nh, unsigned int npaths,
                           bool *is_fdb, struct netlink_ext_ack *extack)
{
        if (nh->is_group) {
                struct nh_group *nhg = rtnl_dereference(nh->nh_grp);

                /* Nesting groups within groups is not supported. */
                if (nhg->hash_threshold) {
                        NL_SET_ERR_MSG(extack,
                                       "Hash-threshold group can not be a nexthop within a group");
                        return false;
                }
                if (nhg->resilient) {
                        NL_SET_ERR_MSG(extack,
                                       "Resilient group can not be a nexthop within a group");
                        return false;
                }
                *is_fdb = nhg->fdb_nh;
        } else {
                struct nh_info *nhi = rtnl_dereference(nh->nh_info);

                if (nhi->reject_nh && npaths > 1) {
                        NL_SET_ERR_MSG(extack,
                                       "Blackhole nexthop can not be used in a group with more than 1 path");
                        return false;
                }
                *is_fdb = nhi->fdb_nh;
        }

        return true;
}

static int nh_check_attr_fdb_group(struct nexthop *nh, u8 *nh_family,
                                   struct netlink_ext_ack *extack)
{
        struct nh_info *nhi;

        nhi = rtnl_dereference(nh->nh_info);

        if (!nhi->fdb_nh) {
                NL_SET_ERR_MSG(extack, "FDB nexthop group can only have fdb nexthops");
                return -EINVAL;
        }

        if (*nh_family == AF_UNSPEC) {
                *nh_family = nhi->family;
        } else if (*nh_family != nhi->family) {
                NL_SET_ERR_MSG(extack, "FDB nexthop group cannot have mixed family nexthops");
                return -EINVAL;
        }

        return 0;
}

static int nh_check_attr_group(struct net *net,
                               struct nlattr *tb[], size_t tb_size,
                               u16 nh_grp_type, struct netlink_ext_ack *extack)
{
        unsigned int len = nla_len(tb[NHA_GROUP]);
        struct nexthop_grp *nhg;
        unsigned int i, j;

        if (!len || len & (sizeof(struct nexthop_grp) - 1)) {
                NL_SET_ERR_MSG(extack,
                               "Invalid length for nexthop group attribute");
                return -EINVAL;
        }

        /* convert len to number of nexthop ids */
        len /= sizeof(*nhg);

        nhg = nla_data(tb[NHA_GROUP]);
        for (i = 0; i < len; ++i) {
                if (nhg[i].resvd2) {
                        NL_SET_ERR_MSG(extack, "Reserved field in nexthop_grp must be 0");
                        return -EINVAL;
                }
                if (nexthop_grp_weight(&nhg[i]) == 0) {
                        /* 0xffff got passed in, representing weight of 0x10000,
                         * which is too heavy.
                         */
                        NL_SET_ERR_MSG(extack, "Invalid value for weight");
                        return -EINVAL;
                }
                for (j = i + 1; j < len; ++j) {
                        if (nhg[i].id == nhg[j].id) {
                                NL_SET_ERR_MSG(extack, "Nexthop id can not be used twice in a group");
                                return -EINVAL;
                        }
                }
        }

        nhg = nla_data(tb[NHA_GROUP]);
        for (i = NHA_GROUP_TYPE + 1; i < tb_size; ++i) {
                if (!tb[i])
                        continue;
                switch (i) {
                case NHA_HW_STATS_ENABLE:
                case NHA_FDB:
                        continue;
                case NHA_RES_GROUP:
                        if (nh_grp_type == NEXTHOP_GRP_TYPE_RES)
                                continue;
                        break;
                }
                NL_SET_ERR_MSG(extack,
                               "No other attributes can be set in nexthop groups");
                return -EINVAL;
        }

        return 0;
}

static int nh_check_attr_group_rtnl(struct net *net, struct nlattr *tb[],
                                    struct netlink_ext_ack *extack)
{
        u8 nh_family = AF_UNSPEC;
        struct nexthop_grp *nhg;
        unsigned int len;
        unsigned int i;
        u8 nhg_fdb;

        len = nla_len(tb[NHA_GROUP]) / sizeof(*nhg);
        nhg = nla_data(tb[NHA_GROUP]);
        nhg_fdb = !!tb[NHA_FDB];

        for (i = 0; i < len; i++) {
                struct nexthop *nh;
                bool is_fdb_nh;

                nh = nexthop_find_by_id(net, nhg[i].id);
                if (!nh) {
                        NL_SET_ERR_MSG(extack, "Invalid nexthop id");
                        return -EINVAL;
                }
                if (!valid_group_nh(nh, len, &is_fdb_nh, extack))
                        return -EINVAL;

                if (nhg_fdb && nh_check_attr_fdb_group(nh, &nh_family, extack))
                        return -EINVAL;

                if (!nhg_fdb && is_fdb_nh) {
                        NL_SET_ERR_MSG(extack, "Non FDB nexthop group cannot have fdb nexthops");
                        return -EINVAL;
                }
        }

        return 0;
}

static bool ipv6_good_nh(const struct fib6_nh *nh)
{
        int state = NUD_REACHABLE;
        struct neighbour *n;

        rcu_read_lock();

        n = __ipv6_neigh_lookup_noref(nh->fib_nh_dev, &nh->fib_nh_gw6);
        if (n)
                state = READ_ONCE(n->nud_state);

        rcu_read_unlock();

        return !!(state & NUD_VALID);
}

static bool ipv4_good_nh(const struct fib_nh *nh)
{
        int state = NUD_REACHABLE;
        struct neighbour *n;

        rcu_read_lock();

        n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
                                      (__force u32)nh->fib_nh_gw4);
        if (n)
                state = READ_ONCE(n->nud_state);

        rcu_read_unlock();

        return !!(state & NUD_VALID);
}

static bool nexthop_is_good_nh(const struct nexthop *nh)
{
        struct nh_info *nhi = rcu_dereference(nh->nh_info);

        switch (nhi->family) {
        case AF_INET:
                return ipv4_good_nh(&nhi->fib_nh);
        case AF_INET6:
                return IS_ENABLED(CONFIG_IPV6) && ipv6_good_nh(&nhi->fib6_nh);
        }

        return false;
}

static struct nexthop *nexthop_select_path_fdb(struct nh_group *nhg, int hash)
{
        int i;

        for (i = 0; i < nhg->num_nh; i++) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                if (hash > atomic_read(&nhge->hthr.upper_bound))
                        continue;

                nh_grp_entry_stats_inc(nhge);
                return nhge->nh;
        }

        WARN_ON_ONCE(1);
        return NULL;
}

static struct nexthop *nexthop_select_path_hthr(struct nh_group *nhg, int hash)
{
        struct nh_grp_entry *nhge0 = NULL;
        int i;

        if (nhg->fdb_nh)
                return nexthop_select_path_fdb(nhg, hash);

        for (i = 0; i < nhg->num_nh; ++i) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                /* nexthops always check if it is good and does
                 * not rely on a sysctl for this behavior
                 */
                if (!nexthop_is_good_nh(nhge->nh))
                        continue;

                if (!nhge0)
                        nhge0 = nhge;

                if (hash > atomic_read(&nhge->hthr.upper_bound))
                        continue;

                nh_grp_entry_stats_inc(nhge);
                return nhge->nh;
        }

        if (!nhge0)
                nhge0 = &nhg->nh_entries[0];
        nh_grp_entry_stats_inc(nhge0);
        return nhge0->nh;
}

static struct nexthop *nexthop_select_path_res(struct nh_group *nhg, int hash)
{
        struct nh_res_table *res_table = rcu_dereference(nhg->res_table);
        u16 bucket_index = hash % res_table->num_nh_buckets;
        struct nh_res_bucket *bucket;
        struct nh_grp_entry *nhge;

        /* nexthop_select_path() is expected to return a non-NULL value, so
         * skip protocol validation and just hand out whatever there is.
         */
        bucket = &res_table->nh_buckets[bucket_index];
        nh_res_bucket_set_busy(bucket);
        nhge = rcu_dereference(bucket->nh_entry);
        nh_grp_entry_stats_inc(nhge);
        return nhge->nh;
}

struct nexthop *nexthop_select_path(struct nexthop *nh, int hash)
{
        struct nh_group *nhg;

        if (!nh->is_group)
                return nh;

        nhg = rcu_dereference(nh->nh_grp);
        if (nhg->hash_threshold)
                return nexthop_select_path_hthr(nhg, hash);
        else if (nhg->resilient)
                return nexthop_select_path_res(nhg, hash);

        /* Unreachable. */
        return NULL;
}
EXPORT_SYMBOL_GPL(nexthop_select_path);

int nexthop_for_each_fib6_nh(struct nexthop *nh,
                             int (*cb)(struct fib6_nh *nh, void *arg),
                             void *arg)
{
        struct nh_info *nhi;
        int err;

        if (nh->is_group) {
                struct nh_group *nhg;
                int i;

                nhg = rcu_dereference_rtnl(nh->nh_grp);
                for (i = 0; i < nhg->num_nh; i++) {
                        struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                        nhi = rcu_dereference_rtnl(nhge->nh->nh_info);
                        err = cb(&nhi->fib6_nh, arg);
                        if (err)
                                return err;
                }
        } else {
                nhi = rcu_dereference_rtnl(nh->nh_info);
                err = cb(&nhi->fib6_nh, arg);
                if (err)
                        return err;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(nexthop_for_each_fib6_nh);

static int check_src_addr(const struct in6_addr *saddr,
                          struct netlink_ext_ack *extack)
{
        if (!ipv6_addr_any(saddr)) {
                NL_SET_ERR_MSG(extack, "IPv6 routes using source address can not use nexthop objects");
                return -EINVAL;
        }
        return 0;
}

int fib6_check_nexthop(struct nexthop *nh, struct fib6_config *cfg,
                       struct netlink_ext_ack *extack)
{
        struct nh_info *nhi;
        bool is_fdb_nh;

        /* fib6_src is unique to a fib6_info and limits the ability to cache
         * routes in fib6_nh within a nexthop that is potentially shared
         * across multiple fib entries. If the config wants to use source
         * routing it can not use nexthop objects. mlxsw also does not allow
         * fib6_src on routes.
         */
        if (cfg && check_src_addr(&cfg->fc_src, extack) < 0)
                return -EINVAL;

        if (nh->is_group) {
                struct nh_group *nhg;

                nhg = rcu_dereference_rtnl(nh->nh_grp);
                if (nhg->has_v4)
                        goto no_v4_nh;
                is_fdb_nh = nhg->fdb_nh;
        } else {
                nhi = rcu_dereference_rtnl(nh->nh_info);
                if (nhi->family == AF_INET)
                        goto no_v4_nh;
                is_fdb_nh = nhi->fdb_nh;
        }

        if (is_fdb_nh) {
                NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
                return -EINVAL;
        }

        return 0;
no_v4_nh:
        NL_SET_ERR_MSG(extack, "IPv6 routes can not use an IPv4 nexthop");
        return -EINVAL;
}
EXPORT_SYMBOL_GPL(fib6_check_nexthop);

/* if existing nexthop has ipv6 routes linked to it, need
 * to verify this new spec works with ipv6
 */
static int fib6_check_nh_list(struct nexthop *old, struct nexthop *new,
                              struct netlink_ext_ack *extack)
{
        struct fib6_info *f6i;
        int err = 0;

        if (list_empty(&old->f6i_list))
                return 0;

        spin_lock_bh(&old->lock);
        list_for_each_entry(f6i, &old->f6i_list, nh_list) {
                err = check_src_addr(&f6i->fib6_src.addr, extack);
                if (err)
                        break;
        }
        spin_unlock_bh(&old->lock);

        if (err)
                return err;

        return fib6_check_nexthop(new, NULL, extack);
}

static int nexthop_check_scope(struct nh_info *nhi, u8 scope,
                               struct netlink_ext_ack *extack)
{
        if (scope == RT_SCOPE_HOST && nhi->fib_nhc.nhc_gw_family) {
                NL_SET_ERR_MSG(extack,
                               "Route with host scope can not have a gateway");
                return -EINVAL;
        }

        if (nhi->fib_nhc.nhc_flags & RTNH_F_ONLINK && scope >= RT_SCOPE_LINK) {
                NL_SET_ERR_MSG(extack, "Scope mismatch with nexthop");
                return -EINVAL;
        }

        return 0;
}

/* Invoked by fib add code to verify nexthop by id is ok with
 * config for prefix; parts of fib_check_nh not done when nexthop
 * object is used.
 */
int fib_check_nexthop(struct nexthop *nh, u8 scope,
                      struct netlink_ext_ack *extack)
{
        struct nh_info *nhi;
        int err = 0;

        if (nh->is_group) {
                struct nh_group *nhg;

                nhg = rtnl_dereference(nh->nh_grp);
                if (nhg->fdb_nh) {
                        NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
                        err = -EINVAL;
                        goto out;
                }

                if (scope == RT_SCOPE_HOST) {
                        NL_SET_ERR_MSG(extack, "Route with host scope can not have multiple nexthops");
                        err = -EINVAL;
                        goto out;
                }

                /* all nexthops in a group have the same scope */
                nhi = rtnl_dereference(nhg->nh_entries[0].nh->nh_info);
                err = nexthop_check_scope(nhi, scope, extack);
        } else {
                nhi = rtnl_dereference(nh->nh_info);
                if (nhi->fdb_nh) {
                        NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
                        err = -EINVAL;
                        goto out;
                }
                err = nexthop_check_scope(nhi, scope, extack);
        }

out:
        return err;
}

static int fib_check_nh_list(struct nexthop *old, struct nexthop *new,
                             struct netlink_ext_ack *extack)
{
        struct fib_info *fi;

        list_for_each_entry(fi, &old->fi_list, nh_list) {
                int err;

                err = fib_check_nexthop(new, fi->fib_scope, extack);
                if (err)
                        return err;
        }
        return 0;
}

static bool nh_res_nhge_is_balanced(const struct nh_grp_entry *nhge)
{
        return nhge->res.count_buckets == nhge->res.wants_buckets;
}

static bool nh_res_nhge_is_ow(const struct nh_grp_entry *nhge)
{
        return nhge->res.count_buckets > nhge->res.wants_buckets;
}

static bool nh_res_nhge_is_uw(const struct nh_grp_entry *nhge)
{
        return nhge->res.count_buckets < nhge->res.wants_buckets;
}

static bool nh_res_table_is_balanced(const struct nh_res_table *res_table)
{
        return list_empty(&res_table->uw_nh_entries);
}

static void nh_res_bucket_unset_nh(struct nh_res_bucket *bucket)
{
        struct nh_grp_entry *nhge;

        if (bucket->occupied) {
                nhge = nh_res_dereference(bucket->nh_entry);
                nhge->res.count_buckets--;
                bucket->occupied = false;
        }
}

static void nh_res_bucket_set_nh(struct nh_res_bucket *bucket,
                                 struct nh_grp_entry *nhge)
{
        nh_res_bucket_unset_nh(bucket);

        bucket->occupied = true;
        rcu_assign_pointer(bucket->nh_entry, nhge);
        nhge->res.count_buckets++;
}

static bool nh_res_bucket_should_migrate(struct nh_res_table *res_table,
                                         struct nh_res_bucket *bucket,
                                         unsigned long *deadline, bool *force)
{
        unsigned long now = jiffies;
        struct nh_grp_entry *nhge;
        unsigned long idle_point;

        if (!bucket->occupied) {
                /* The bucket is not occupied, its NHGE pointer is either
                 * NULL or obsolete. We _have to_ migrate: set force.
                 */
                *force = true;
                return true;
        }

        nhge = nh_res_dereference(bucket->nh_entry);

        /* If the bucket is populated by an underweight or balanced
         * nexthop, do not migrate.
         */
        if (!nh_res_nhge_is_ow(nhge))
                return false;

        /* At this point we know that the bucket is populated with an
         * overweight nexthop. It needs to be migrated to a new nexthop if
         * the idle timer of unbalanced timer expired.
         */

        idle_point = nh_res_bucket_idle_point(res_table, bucket, now);
        if (time_after_eq(now, idle_point)) {
                /* The bucket is idle. We _can_ migrate: unset force. */
                *force = false;
                return true;
        }

        /* Unbalanced timer of 0 means "never force". */
        if (res_table->unbalanced_timer) {
                unsigned long unb_point;

                unb_point = nh_res_table_unb_point(res_table);
                if (time_after(now, unb_point)) {
                        /* The bucket is not idle, but the unbalanced timer
                         * expired. We _can_ migrate, but set force anyway,
                         * so that drivers know to ignore activity reports
                         * from the HW.
                         */
                        *force = true;
                        return true;
                }

                nh_res_time_set_deadline(unb_point, deadline);
        }

        nh_res_time_set_deadline(idle_point, deadline);
        return false;
}

static bool nh_res_bucket_migrate(struct nh_res_table *res_table,
                                  u16 bucket_index, bool notify,
                                  bool notify_nl, bool force)
{
        struct nh_res_bucket *bucket = &res_table->nh_buckets[bucket_index];
        struct netlink_ext_ack extack = {};
        struct nh_grp_entry *new_nhge;
        int err;

        new_nhge = list_first_entry_or_null(&res_table->uw_nh_entries,
                                            struct nh_grp_entry,
                                            res.uw_nh_entry);
        if (WARN_ON_ONCE(!new_nhge))
                /* If this function is called, "bucket" is either not
                 * occupied, or it belongs to a next hop that is
                 * overweight. In either case, there ought to be a
                 * corresponding underweight next hop.
                 */
                return false;

        if (notify) {
                struct nh_grp_entry *old_nhge;

                old_nhge = nh_res_dereference(bucket->nh_entry);
                err = call_nexthop_res_bucket_notifiers(res_table->net,
                                                        res_table->nhg_id,
                                                        bucket_index, force,
                                                        old_nhge->nh,
                                                        new_nhge->nh, &extack);
                if (err) {
                        pr_err_ratelimited("%s\n", extack._msg);
                        if (!force)
                                return false;
                        /* It is not possible to veto a forced replacement, so
                         * just clear the hardware flags from the nexthop
                         * bucket to indicate to user space that this bucket is
                         * not correctly populated in hardware.
                         */
                        bucket->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
                }
        }

        nh_res_bucket_set_nh(bucket, new_nhge);
        nh_res_bucket_set_idle(res_table, bucket);

        if (notify_nl)
                nexthop_bucket_notify(res_table, bucket_index);

        if (nh_res_nhge_is_balanced(new_nhge))
                list_del(&new_nhge->res.uw_nh_entry);
        return true;
}

#define NH_RES_UPKEEP_DW_MINIMUM_INTERVAL (HZ / 2)

static void nh_res_table_upkeep(struct nh_res_table *res_table,
                                bool notify, bool notify_nl)
{
        unsigned long now = jiffies;
        unsigned long deadline;
        u16 i;

        /* Deadline is the next time that upkeep should be run. It is the
         * earliest time at which one of the buckets might be migrated.
         * Start at the most pessimistic estimate: either unbalanced_timer
         * from now, or if there is none, idle_timer from now. For each
         * encountered time point, call nh_res_time_set_deadline() to
         * refine the estimate.
         */
        if (res_table->unbalanced_timer)
                deadline = now + res_table->unbalanced_timer;
        else
                deadline = now + res_table->idle_timer;

        for (i = 0; i < res_table->num_nh_buckets; i++) {
                struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
                bool force;

                if (nh_res_bucket_should_migrate(res_table, bucket,
                                                 &deadline, &force)) {
                        if (!nh_res_bucket_migrate(res_table, i, notify,
                                                   notify_nl, force)) {
                                unsigned long idle_point;

                                /* A driver can override the migration
                                 * decision if the HW reports that the
                                 * bucket is actually not idle. Therefore
                                 * remark the bucket as busy again and
                                 * update the deadline.
                                 */
                                nh_res_bucket_set_busy(bucket);
                                idle_point = nh_res_bucket_idle_point(res_table,
                                                                      bucket,
                                                                      now);
                                nh_res_time_set_deadline(idle_point, &deadline);
                        }
                }
        }

        /* If the group is still unbalanced, schedule the next upkeep to
         * either the deadline computed above, or the minimum deadline,
         * whichever comes later.
         */
        if (!nh_res_table_is_balanced(res_table)) {
                unsigned long now = jiffies;
                unsigned long min_deadline;

                min_deadline = now + NH_RES_UPKEEP_DW_MINIMUM_INTERVAL;
                if (time_before(deadline, min_deadline))
                        deadline = min_deadline;

                queue_delayed_work(system_power_efficient_wq,
                                   &res_table->upkeep_dw, deadline - now);
        }
}

static void nh_res_table_upkeep_dw(struct work_struct *work)
{
        struct delayed_work *dw = to_delayed_work(work);
        struct nh_res_table *res_table;

        res_table = container_of(dw, struct nh_res_table, upkeep_dw);
        nh_res_table_upkeep(res_table, true, true);
}

static void nh_res_table_cancel_upkeep(struct nh_res_table *res_table)
{
        cancel_delayed_work_sync(&res_table->upkeep_dw);
}

static void nh_res_group_rebalance(struct nh_group *nhg,
                                   struct nh_res_table *res_table)
{
        u16 prev_upper_bound = 0;
        u32 total = 0;
        u32 w = 0;
        int i;

        INIT_LIST_HEAD(&res_table->uw_nh_entries);

        for (i = 0; i < nhg->num_nh; ++i)
                total += nhg->nh_entries[i].weight;

        for (i = 0; i < nhg->num_nh; ++i) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];
                u16 upper_bound;
                u64 btw;

                w += nhge->weight;
                btw = ((u64)res_table->num_nh_buckets) * w;
                upper_bound = DIV_ROUND_CLOSEST_ULL(btw, total);
                nhge->res.wants_buckets = upper_bound - prev_upper_bound;
                prev_upper_bound = upper_bound;

                if (nh_res_nhge_is_uw(nhge)) {
                        if (list_empty(&res_table->uw_nh_entries))
                                res_table->unbalanced_since = jiffies;
                        list_add(&nhge->res.uw_nh_entry,
                                 &res_table->uw_nh_entries);
                }
        }
}

/* Migrate buckets in res_table so that they reference NHGE's from NHG with
 * the right NH ID. Set those buckets that do not have a corresponding NHGE
 * entry in NHG as not occupied.
 */
static void nh_res_table_migrate_buckets(struct nh_res_table *res_table,
                                         struct nh_group *nhg)
{
        u16 i;

        for (i = 0; i < res_table->num_nh_buckets; i++) {
                struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
                u32 id = rtnl_dereference(bucket->nh_entry)->nh->id;
                bool found = false;
                int j;

                for (j = 0; j < nhg->num_nh; j++) {
                        struct nh_grp_entry *nhge = &nhg->nh_entries[j];

                        if (nhge->nh->id == id) {
                                nh_res_bucket_set_nh(bucket, nhge);
                                found = true;
                                break;
                        }
                }

                if (!found)
                        nh_res_bucket_unset_nh(bucket);
        }
}

static void replace_nexthop_grp_res(struct nh_group *oldg,
                                    struct nh_group *newg)
{
        /* For NH group replacement, the new NHG might only have a stub
         * hash table with 0 buckets, because the number of buckets was not
         * specified. For NH removal, oldg and newg both reference the same
         * res_table. So in any case, in the following, we want to work
         * with oldg->res_table.
         */
        struct nh_res_table *old_res_table = rtnl_dereference(oldg->res_table);
        unsigned long prev_unbalanced_since = old_res_table->unbalanced_since;
        bool prev_has_uw = !list_empty(&old_res_table->uw_nh_entries);

        nh_res_table_cancel_upkeep(old_res_table);
        nh_res_table_migrate_buckets(old_res_table, newg);
        nh_res_group_rebalance(newg, old_res_table);
        if (prev_has_uw && !list_empty(&old_res_table->uw_nh_entries))
                old_res_table->unbalanced_since = prev_unbalanced_since;
        nh_res_table_upkeep(old_res_table, true, false);
}

static void nh_hthr_group_rebalance(struct nh_group *nhg)
{
        u32 total = 0;
        u32 w = 0;
        int i;

        for (i = 0; i < nhg->num_nh; ++i)
                total += nhg->nh_entries[i].weight;

        for (i = 0; i < nhg->num_nh; ++i) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];
                u32 upper_bound;

                w += nhge->weight;
                upper_bound = DIV_ROUND_CLOSEST_ULL((u64)w << 31, total) - 1;
                atomic_set(&nhge->hthr.upper_bound, upper_bound);
        }
}

static bool __must_check
remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
                    struct nl_info *nlinfo, struct list_head *deferred_free)
{
        struct nh_grp_entry *nhges, *new_nhges;
        struct nexthop *nhp = nhge->nh_parent;
        struct netlink_ext_ack extack;
        struct nexthop *nh = nhge->nh;
        struct nh_group *nhg, *newg;
        int i, j, err;

        WARN_ON(!nh);

        nhg = rtnl_dereference(nhp->nh_grp);
        newg = nhg->spare;

        /* last entry, keep it visible and remove the parent */
        if (nhg->num_nh == 1)
                return remove_nexthop(net, nhp, nlinfo);

        newg->has_v4 = false;
        newg->is_multipath = nhg->is_multipath;
        newg->hash_threshold = nhg->hash_threshold;
        newg->resilient = nhg->resilient;
        newg->fdb_nh = nhg->fdb_nh;
        newg->num_nh = nhg->num_nh;

        /* copy old entries to new except the one getting removed */
        nhges = nhg->nh_entries;
        new_nhges = newg->nh_entries;
        for (i = 0, j = 0; i < nhg->num_nh; ++i) {
                struct nh_info *nhi;

                /* current nexthop getting removed */
                if (nhg->nh_entries[i].nh == nh) {
                        newg->num_nh--;
                        continue;
                }

                nhi = rtnl_dereference(nhges[i].nh->nh_info);
                if (nhi->family == AF_INET)
                        newg->has_v4 = true;

                list_del(&nhges[i].nh_list);
                new_nhges[j].stats = nhges[i].stats;
                new_nhges[j].nh_parent = nhges[i].nh_parent;
                new_nhges[j].nh = nhges[i].nh;
                new_nhges[j].weight = nhges[i].weight;
                list_add(&new_nhges[j].nh_list, &new_nhges[j].nh->grp_list);
                j++;
        }

        if (newg->hash_threshold)
                nh_hthr_group_rebalance(newg);
        else if (newg->resilient)
                replace_nexthop_grp_res(nhg, newg);

        rcu_assign_pointer(nhp->nh_grp, newg);

        list_del(&nhge->nh_list);
        nexthop_put(nhge->nh);
        list_add(&nhge->nh_list, deferred_free);

        /* Removal of a NH from a resilient group is notified through
         * bucket notifications.
         */
        if (newg->hash_threshold) {
                err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, nhp,
                                             &extack);
                if (err)
                        pr_err("%s\n", extack._msg);
        }

        if (nlinfo)
                nexthop_notify(RTM_NEWNEXTHOP, nhp, nlinfo);

        return false;
}

static bool __must_check
remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
                           struct nl_info *nlinfo)
{
        struct nh_grp_entry *nhge, *tmp;
        LIST_HEAD(deferred_free);
        bool need_flush = false;

        /* If there is nothing to do, let's avoid the costly call to
         * synchronize_net()
         */
        if (list_empty(&nh->grp_list))
                return false;

        list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list)
                need_flush |= remove_nh_grp_entry(net, nhge, nlinfo,
                                                  &deferred_free);

        /* make sure all see the newly published array before releasing rtnl */
        synchronize_net();

        /* Now safe to free percpu stats — all RCU readers have finished */
        list_for_each_entry_safe(nhge, tmp, &deferred_free, nh_list) {
                list_del(&nhge->nh_list);
                free_percpu(nhge->stats);
        }

        return need_flush;
}

static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)
{
        struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);
        struct nh_res_table *res_table;
        int i, num_nh = nhg->num_nh;

        for (i = 0; i < num_nh; ++i) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                if (WARN_ON(!nhge->nh))
                        continue;

                list_del_init(&nhge->nh_list);
        }

        if (nhg->resilient) {
                res_table = rtnl_dereference(nhg->res_table);
                nh_res_table_cancel_upkeep(res_table);
        }
}

/* not called for nexthop replace */
static bool __must_check __remove_nexthop_fib(struct net *net,
                                              struct nexthop *nh)
{
        bool need_flush = !list_empty(&nh->fi_list);
        struct fib6_info *f6i;
        struct fib_info *fi;

        list_for_each_entry(fi, &nh->fi_list, nh_list)
                fi->fib_flags |= RTNH_F_DEAD;

        spin_lock_bh(&nh->lock);

        nh->dead = true;

        while (!list_empty(&nh->f6i_list)) {
                f6i = list_first_entry(&nh->f6i_list, typeof(*f6i), nh_list);

                /* __ip6_del_rt does a release, so do a hold here */
                fib6_info_hold(f6i);

                spin_unlock_bh(&nh->lock);
                ip6_del_rt(net, f6i,
                           !READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode));

                spin_lock_bh(&nh->lock);
        }

        spin_unlock_bh(&nh->lock);

        return need_flush;
}

static bool __must_check __remove_nexthop(struct net *net, struct nexthop *nh,
                                          struct nl_info *nlinfo)
{
        bool need_flush = __remove_nexthop_fib(net, nh);

        if (nh->is_group) {
                remove_nexthop_group(nh, nlinfo);
        } else {
                struct nh_info *nhi;

                nhi = rtnl_dereference(nh->nh_info);
                if (nhi->fib_nhc.nhc_dev)
                        hlist_del(&nhi->dev_hash);

                need_flush |= remove_nexthop_from_groups(net, nh, nlinfo);
        }

        return need_flush;
}

static bool __must_check remove_nexthop(struct net *net, struct nexthop *nh,
                                        struct nl_info *nlinfo)
{
        bool need_flush;

        call_nexthop_notifiers(net, NEXTHOP_EVENT_DEL, nh, NULL);

        /* remove from the tree */
        rb_erase(&nh->rb_node, &net->nexthop.rb_root);

        if (nlinfo)
                nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo);

        need_flush = __remove_nexthop(net, nh, nlinfo);
        nh_base_seq_inc(net);

        nexthop_put(nh);

        return need_flush;
}

static void remove_one_nexthop(struct net *net, struct nexthop *nh,
                               struct nl_info *nlinfo)
{
        if (remove_nexthop(net, nh, nlinfo))
                fib_flush(net);
}

/* if any FIB entries reference this nexthop, any dst entries
 * need to be regenerated
 */
static void nh_rt_cache_flush(struct net *net, struct nexthop *nh,
                              struct nexthop *replaced_nh)
{
        struct nh_group *nhg;
        bool have_f6i;
        int i;

        if (!list_empty(&nh->fi_list))
                rt_cache_flush(net);

        spin_lock_bh(&nh->lock);
        have_f6i = !list_empty(&nh->f6i_list);
        spin_unlock_bh(&nh->lock);
        if (have_f6i)
                rt_genid_bump_ipv6(net);

        /* if an IPv6 group was replaced, we have to release all old
         * dsts to make sure all refcounts are released
         */
        if (!replaced_nh->is_group)
                return;

        nhg = rtnl_dereference(replaced_nh->nh_grp);
        for (i = 0; i < nhg->num_nh; i++) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];
                struct nh_info *nhi = rtnl_dereference(nhge->nh->nh_info);

                if (nhi->family == AF_INET6)
                        fib6_nh_release_dsts(&nhi->fib6_nh);
        }
}

static int replace_nexthop_grp(struct net *net, struct nexthop *old,
                               struct nexthop *new, const struct nh_config *cfg,
                               struct netlink_ext_ack *extack)
{
        struct nh_res_table *tmp_table = NULL;
        struct nh_res_table *new_res_table;
        struct nh_res_table *old_res_table;
        struct nh_group *oldg, *newg;
        int i, err;

        if (!new->is_group) {
                NL_SET_ERR_MSG(extack, "Can not replace a nexthop group with a nexthop.");
                return -EINVAL;
        }

        oldg = rtnl_dereference(old->nh_grp);
        newg = rtnl_dereference(new->nh_grp);

        if (newg->hash_threshold != oldg->hash_threshold) {
                NL_SET_ERR_MSG(extack, "Can not replace a nexthop group with one of a different type.");
                return -EINVAL;
        }

        if (newg->hash_threshold) {
                err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new,
                                             extack);
                if (err)
                        return err;
        } else if (newg->resilient) {
                new_res_table = rtnl_dereference(newg->res_table);
                old_res_table = rtnl_dereference(oldg->res_table);

                /* Accept if num_nh_buckets was not given, but if it was
                 * given, demand that the value be correct.
                 */
                if (cfg->nh_grp_res_has_num_buckets &&
                    cfg->nh_grp_res_num_buckets !=
                    old_res_table->num_nh_buckets) {
                        NL_SET_ERR_MSG(extack, "Can not change number of buckets of a resilient nexthop group.");
                        return -EINVAL;
                }

                /* Emit a pre-replace notification so that listeners could veto
                 * a potentially unsupported configuration. Otherwise,
                 * individual bucket replacement notifications would need to be
                 * vetoed, which is something that should only happen if the
                 * bucket is currently active.
                 */
                err = call_nexthop_res_table_notifiers(net, new, extack);
                if (err)
                        return err;

                if (cfg->nh_grp_res_has_idle_timer)
                        old_res_table->idle_timer = cfg->nh_grp_res_idle_timer;
                if (cfg->nh_grp_res_has_unbalanced_timer)
                        old_res_table->unbalanced_timer =
                                cfg->nh_grp_res_unbalanced_timer;

                replace_nexthop_grp_res(oldg, newg);

                tmp_table = new_res_table;
                rcu_assign_pointer(newg->res_table, old_res_table);
                rcu_assign_pointer(newg->spare->res_table, old_res_table);
        }

        /* update parents - used by nexthop code for cleanup */
        for (i = 0; i < newg->num_nh; i++)
                newg->nh_entries[i].nh_parent = old;

        rcu_assign_pointer(old->nh_grp, newg);

        /* Make sure concurrent readers are not using 'oldg' anymore. */
        synchronize_net();

        if (newg->resilient) {
                rcu_assign_pointer(oldg->res_table, tmp_table);
                rcu_assign_pointer(oldg->spare->res_table, tmp_table);
        }

        for (i = 0; i < oldg->num_nh; i++)
                oldg->nh_entries[i].nh_parent = new;

        rcu_assign_pointer(new->nh_grp, oldg);

        return 0;
}

static void nh_group_v4_update(struct nh_group *nhg)
{
        struct nh_grp_entry *nhges;
        bool has_v4 = false;
        int i;

        nhges = nhg->nh_entries;
        for (i = 0; i < nhg->num_nh; i++) {
                struct nh_info *nhi;

                nhi = rtnl_dereference(nhges[i].nh->nh_info);
                if (nhi->family == AF_INET)
                        has_v4 = true;
        }
        nhg->has_v4 = has_v4;
}

static int replace_nexthop_single_notify_res(struct net *net,
                                             struct nh_res_table *res_table,
                                             struct nexthop *old,
                                             struct nh_info *oldi,
                                             struct nh_info *newi,
                                             struct netlink_ext_ack *extack)
{
        u32 nhg_id = res_table->nhg_id;
        int err;
        u16 i;

        for (i = 0; i < res_table->num_nh_buckets; i++) {
                struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
                struct nh_grp_entry *nhge;

                nhge = rtnl_dereference(bucket->nh_entry);
                if (nhge->nh == old) {
                        err = __call_nexthop_res_bucket_notifiers(net, nhg_id,
                                                                  i, true,
                                                                  oldi, newi,
                                                                  extack);
                        if (err)
                                goto err_notify;
                }
        }

        return 0;

err_notify:
        while (i-- > 0) {
                struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
                struct nh_grp_entry *nhge;

                nhge = rtnl_dereference(bucket->nh_entry);
                if (nhge->nh == old)
                        __call_nexthop_res_bucket_notifiers(net, nhg_id, i,
                                                            true, newi, oldi,
                                                            extack);
        }
        return err;
}

static int replace_nexthop_single_notify(struct net *net,
                                         struct nexthop *group_nh,
                                         struct nexthop *old,
                                         struct nh_info *oldi,
                                         struct nh_info *newi,
                                         struct netlink_ext_ack *extack)
{
        struct nh_group *nhg = rtnl_dereference(group_nh->nh_grp);
        struct nh_res_table *res_table;

        if (nhg->hash_threshold) {
                return call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE,
                                              group_nh, extack);
        } else if (nhg->resilient) {
                res_table = rtnl_dereference(nhg->res_table);
                return replace_nexthop_single_notify_res(net, res_table,
                                                         old, oldi, newi,
                                                         extack);
        }

        return -EINVAL;
}

static int replace_nexthop_single(struct net *net, struct nexthop *old,
                                  struct nexthop *new,
                                  struct netlink_ext_ack *extack)
{
        u8 old_protocol, old_nh_flags;
        struct nh_info *oldi, *newi;
        struct nh_grp_entry *nhge;
        int err;

        if (new->is_group) {
                NL_SET_ERR_MSG(extack, "Can not replace a nexthop with a nexthop group.");
                return -EINVAL;
        }

        if (!list_empty(&old->grp_list) &&
            rtnl_dereference(new->nh_info)->fdb_nh !=
            rtnl_dereference(old->nh_info)->fdb_nh) {
                NL_SET_ERR_MSG(extack, "Cannot change nexthop FDB status while in a group");
                return -EINVAL;
        }

        err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new, extack);
        if (err)
                return err;

        /* Hardware flags were set on 'old' as 'new' is not in the red-black
         * tree. Therefore, inherit the flags from 'old' to 'new'.
         */
        new->nh_flags |= old->nh_flags & (RTNH_F_OFFLOAD | RTNH_F_TRAP);

        oldi = rtnl_dereference(old->nh_info);
        newi = rtnl_dereference(new->nh_info);

        newi->nh_parent = old;
        oldi->nh_parent = new;

        old_protocol = old->protocol;
        old_nh_flags = old->nh_flags;

        old->protocol = new->protocol;
        old->nh_flags = new->nh_flags;

        rcu_assign_pointer(old->nh_info, newi);
        rcu_assign_pointer(new->nh_info, oldi);

        /* Send a replace notification for all the groups using the nexthop. */
        list_for_each_entry(nhge, &old->grp_list, nh_list) {
                struct nexthop *nhp = nhge->nh_parent;

                err = replace_nexthop_single_notify(net, nhp, old, oldi, newi,
                                                    extack);
                if (err)
                        goto err_notify;
        }

        /* When replacing a nexthop with one of a different family, potentially
         * update IPv4 indication in all the groups using the nexthop.
         */
        if (oldi->family != newi->family) {
                list_for_each_entry(nhge, &old->grp_list, nh_list) {
                        struct nexthop *nhp = nhge->nh_parent;
                        struct nh_group *nhg;

                        nhg = rtnl_dereference(nhp->nh_grp);
                        nh_group_v4_update(nhg);
                }
        }

        return 0;

err_notify:
        rcu_assign_pointer(new->nh_info, newi);
        rcu_assign_pointer(old->nh_info, oldi);
        old->nh_flags = old_nh_flags;
        old->protocol = old_protocol;
        oldi->nh_parent = old;
        newi->nh_parent = new;
        list_for_each_entry_continue_reverse(nhge, &old->grp_list, nh_list) {
                struct nexthop *nhp = nhge->nh_parent;

                replace_nexthop_single_notify(net, nhp, old, newi, oldi, NULL);
        }
        call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, old, extack);
        return err;
}

static void __nexthop_replace_notify(struct net *net, struct nexthop *nh,
                                     struct nl_info *info)
{
        struct fib6_info *f6i;

        if (!list_empty(&nh->fi_list)) {
                struct fib_info *fi;

                /* expectation is a few fib_info per nexthop and then
                 * a lot of routes per fib_info. So mark the fib_info
                 * and then walk the fib tables once
                 */
                list_for_each_entry(fi, &nh->fi_list, nh_list)
                        fi->nh_updated = true;

                fib_info_notify_update(net, info);

                list_for_each_entry(fi, &nh->fi_list, nh_list)
                        fi->nh_updated = false;
        }

        spin_lock_bh(&nh->lock);
        list_for_each_entry(f6i, &nh->f6i_list, nh_list)
                fib6_rt_update(net, f6i, info);
        spin_unlock_bh(&nh->lock);
}

/* send RTM_NEWROUTE with REPLACE flag set for all FIB entries
 * linked to this nexthop and for all groups that the nexthop
 * is a member of
 */
static void nexthop_replace_notify(struct net *net, struct nexthop *nh,
                                   struct nl_info *info)
{
        struct nh_grp_entry *nhge;

        __nexthop_replace_notify(net, nh, info);

        list_for_each_entry(nhge, &nh->grp_list, nh_list)
                __nexthop_replace_notify(net, nhge->nh_parent, info);
}

static int replace_nexthop(struct net *net, struct nexthop *old,
                           struct nexthop *new, const struct nh_config *cfg,
                           struct netlink_ext_ack *extack)
{
        bool new_is_reject = false;
        struct nh_grp_entry *nhge;
        int err;

        /* check that existing FIB entries are ok with the
         * new nexthop definition
         */
        err = fib_check_nh_list(old, new, extack);
        if (err)
                return err;

        err = fib6_check_nh_list(old, new, extack);
        if (err)
                return err;

        if (!new->is_group) {
                struct nh_info *nhi = rtnl_dereference(new->nh_info);

                new_is_reject = nhi->reject_nh;
        }

        list_for_each_entry(nhge, &old->grp_list, nh_list) {
                /* if new nexthop is a blackhole, any groups using this
                 * nexthop cannot have more than 1 path
                 */
                if (new_is_reject &&
                    nexthop_num_path(nhge->nh_parent) > 1) {
                        NL_SET_ERR_MSG(extack, "Blackhole nexthop can not be a member of a group with more than one path");
                        return -EINVAL;
                }

                err = fib_check_nh_list(nhge->nh_parent, new, extack);
                if (err)
                        return err;

                err = fib6_check_nh_list(nhge->nh_parent, new, extack);
                if (err)
                        return err;
        }

        if (old->is_group)
                err = replace_nexthop_grp(net, old, new, cfg, extack);
        else
                err = replace_nexthop_single(net, old, new, extack);

        if (!err) {
                nh_rt_cache_flush(net, old, new);

                WARN_ON_ONCE(__remove_nexthop(net, new, NULL));
                nexthop_put(new);
        }

        return err;
}

/* called with rtnl_lock held */
static int insert_nexthop(struct net *net, struct nexthop *new_nh,
                          struct nh_config *cfg, struct netlink_ext_ack *extack)
{
        struct rb_node **pp, *parent = NULL, *next;
        struct rb_root *root = &net->nexthop.rb_root;
        bool replace = !!(cfg->nlflags & NLM_F_REPLACE);
        bool create = !!(cfg->nlflags & NLM_F_CREATE);
        u32 new_id = new_nh->id;
        int replace_notify = 0;
        int rc = -EEXIST;

        pp = &root->rb_node;
        while (1) {
                struct nexthop *nh;

                next = *pp;
                if (!next)
                        break;

                parent = next;

                nh = rb_entry(parent, struct nexthop, rb_node);
                if (new_id < nh->id) {
                        pp = &next->rb_left;
                } else if (new_id > nh->id) {
                        pp = &next->rb_right;
                } else if (replace) {
                        rc = replace_nexthop(net, nh, new_nh, cfg, extack);
                        if (!rc) {
                                new_nh = nh; /* send notification with old nh */
                                replace_notify = 1;
                        }
                        goto out;
                } else {
                        /* id already exists and not a replace */
                        goto out;
                }
        }

        if (replace && !create) {
                NL_SET_ERR_MSG(extack, "Replace specified without create and no entry exists");
                rc = -ENOENT;
                goto out;
        }

        if (new_nh->is_group) {
                struct nh_group *nhg = rtnl_dereference(new_nh->nh_grp);
                struct nh_res_table *res_table;

                if (nhg->resilient) {
                        res_table = rtnl_dereference(nhg->res_table);

                        /* Not passing the number of buckets is OK when
                         * replacing, but not when creating a new group.
                         */
                        if (!cfg->nh_grp_res_has_num_buckets) {
                                NL_SET_ERR_MSG(extack, "Number of buckets not specified for nexthop group insertion");
                                rc = -EINVAL;
                                goto out;
                        }

                        nh_res_group_rebalance(nhg, res_table);

                        /* Do not send bucket notifications, we do full
                         * notification below.
                         */
                        nh_res_table_upkeep(res_table, false, false);
                }
        }

        rb_link_node_rcu(&new_nh->rb_node, parent, pp);
        rb_insert_color(&new_nh->rb_node, root);

        /* The initial insertion is a full notification for hash-threshold as
         * well as resilient groups.
         */
        rc = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new_nh, extack);
        if (rc)
                rb_erase(&new_nh->rb_node, &net->nexthop.rb_root);

out:
        if (!rc) {
                nh_base_seq_inc(net);
                nexthop_notify(RTM_NEWNEXTHOP, new_nh, &cfg->nlinfo);
                if (replace_notify &&
                    READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode))
                        nexthop_replace_notify(net, new_nh, &cfg->nlinfo);
        }

        return rc;
}

/* rtnl */
/* remove all nexthops tied to a device being deleted */
static void nexthop_flush_dev(struct net_device *dev, unsigned long event)
{
        unsigned int hash = nh_dev_hashfn(dev->ifindex);
        struct net *net = dev_net(dev);
        struct hlist_head *head = &net->nexthop.devhash[hash];
        bool need_flush = false;
        struct hlist_node *n;
        struct nh_info *nhi;

        hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
                if (nhi->fib_nhc.nhc_dev != dev)
                        continue;

                if (nhi->reject_nh &&
                    (event == NETDEV_DOWN || event == NETDEV_CHANGE))
                        continue;

                need_flush |= remove_nexthop(net, nhi->nh_parent, NULL);
        }

        if (need_flush)
                fib_flush(net);
}

/* rtnl; called when net namespace is deleted */
static void flush_all_nexthops(struct net *net)
{
        struct rb_root *root = &net->nexthop.rb_root;
        bool need_flush = false;
        struct rb_node *node;
        struct nexthop *nh;

        while ((node = rb_first(root))) {
                nh = rb_entry(node, struct nexthop, rb_node);
                need_flush |= remove_nexthop(net, nh, NULL);
                cond_resched();
        }
        if (need_flush)
                fib_flush(net);
}

static struct nexthop *nexthop_create_group(struct net *net,
                                            struct nh_config *cfg)
{
        struct nlattr *grps_attr = cfg->nh_grp;
        struct nexthop_grp *entry = nla_data(grps_attr);
        u16 num_nh = nla_len(grps_attr) / sizeof(*entry);
        struct nh_group *nhg;
        struct nexthop *nh;
        int err;
        int i;

        nh = nexthop_alloc();
        if (!nh)
                return ERR_PTR(-ENOMEM);

        nh->is_group = 1;

        nhg = nexthop_grp_alloc(num_nh);
        if (!nhg) {
                kfree(nh);
                return ERR_PTR(-ENOMEM);
        }

        /* spare group used for removals */
        nhg->spare = nexthop_grp_alloc(num_nh);
        if (!nhg->spare) {
                kfree(nhg);
                kfree(nh);
                return ERR_PTR(-ENOMEM);
        }
        nhg->spare->spare = nhg;

        for (i = 0; i < nhg->num_nh; ++i) {
                struct nexthop *nhe;
                struct nh_info *nhi;

                nhe = nexthop_find_by_id(net, entry[i].id);
                if (!nexthop_get(nhe)) {
                        err = -ENOENT;
                        goto out_no_nh;
                }

                nhi = rtnl_dereference(nhe->nh_info);
                if (nhi->family == AF_INET)
                        nhg->has_v4 = true;

                nhg->nh_entries[i].stats =
                        netdev_alloc_pcpu_stats(struct nh_grp_entry_stats);
                if (!nhg->nh_entries[i].stats) {
                        err = -ENOMEM;
                        nexthop_put(nhe);
                        goto out_no_nh;
                }
                nhg->nh_entries[i].nh = nhe;
                nhg->nh_entries[i].weight = nexthop_grp_weight(&entry[i]);

                list_add(&nhg->nh_entries[i].nh_list, &nhe->grp_list);
                nhg->nh_entries[i].nh_parent = nh;
        }

        if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_MPATH) {
                nhg->hash_threshold = 1;
                nhg->is_multipath = true;
        } else if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_RES) {
                struct nh_res_table *res_table;

                res_table = nexthop_res_table_alloc(net, cfg->nh_id, cfg);
                if (!res_table) {
                        err = -ENOMEM;
                        goto out_no_nh;
                }

                rcu_assign_pointer(nhg->spare->res_table, res_table);
                rcu_assign_pointer(nhg->res_table, res_table);
                nhg->resilient = true;
                nhg->is_multipath = true;
        }

        WARN_ON_ONCE(nhg->hash_threshold + nhg->resilient != 1);

        if (nhg->hash_threshold)
                nh_hthr_group_rebalance(nhg);

        if (cfg->nh_fdb)
                nhg->fdb_nh = 1;

        if (cfg->nh_hw_stats)
                nhg->hw_stats = true;

        rcu_assign_pointer(nh->nh_grp, nhg);

        return nh;

out_no_nh:
        for (i--; i >= 0; --i) {
                list_del(&nhg->nh_entries[i].nh_list);
                free_percpu(nhg->nh_entries[i].stats);
                nexthop_put(nhg->nh_entries[i].nh);
        }

        kfree(nhg->spare);
        kfree(nhg);
        kfree(nh);

        return ERR_PTR(err);
}

static int nh_create_ipv4(struct net *net, struct nexthop *nh,
                          struct nh_info *nhi, struct nh_config *cfg,
                          struct netlink_ext_ack *extack)
{
        struct fib_nh *fib_nh = &nhi->fib_nh;
        struct fib_config fib_cfg = {
                .fc_oif   = cfg->nh_ifindex,
                .fc_gw4   = cfg->gw.ipv4,
                .fc_gw_family = cfg->gw.ipv4 ? AF_INET : 0,
                .fc_flags = cfg->nh_flags,
                .fc_nlinfo = cfg->nlinfo,
                .fc_encap = cfg->nh_encap,
                .fc_encap_type = cfg->nh_encap_type,
        };
        u32 tb_id = (cfg->dev ? l3mdev_fib_table(cfg->dev) : RT_TABLE_MAIN);
        int err;

        err = fib_nh_init(net, fib_nh, &fib_cfg, 1, extack);
        if (err) {
                fib_nh_release(net, fib_nh);
                goto out;
        }

        if (nhi->fdb_nh)
                goto out;

        /* sets nh_dev if successful */
        err = fib_check_nh(net, fib_nh, tb_id, 0, extack);
        if (!err) {
                nh->nh_flags = fib_nh->fib_nh_flags;
                fib_info_update_nhc_saddr(net, &fib_nh->nh_common,
                                          !fib_nh->fib_nh_scope ? 0 : fib_nh->fib_nh_scope - 1);
        } else {
                fib_nh_release(net, fib_nh);
        }
out:
        return err;
}

static int nh_create_ipv6(struct net *net,  struct nexthop *nh,
                          struct nh_info *nhi, struct nh_config *cfg,
                          struct netlink_ext_ack *extack)
{
        struct fib6_nh *fib6_nh = &nhi->fib6_nh;
        struct fib6_config fib6_cfg = {
                .fc_table = l3mdev_fib_table(cfg->dev),
                .fc_ifindex = cfg->nh_ifindex,
                .fc_gateway = cfg->gw.ipv6,
                .fc_flags = cfg->nh_flags,
                .fc_nlinfo = cfg->nlinfo,
                .fc_encap = cfg->nh_encap,
                .fc_encap_type = cfg->nh_encap_type,
                .fc_is_fdb = cfg->nh_fdb,
        };
        int err;

        if (!ipv6_addr_any(&cfg->gw.ipv6))
                fib6_cfg.fc_flags |= RTF_GATEWAY;

        /* sets nh_dev if successful */
        err = fib6_nh_init(net, fib6_nh, &fib6_cfg, GFP_KERNEL, extack);
        if (err) {
                /* IPv6 is not enabled, don't call fib6_nh_release */
                if (err == -EAFNOSUPPORT)
                        goto out;
                fib6_nh_release(fib6_nh);
        } else {
                nh->nh_flags = fib6_nh->fib_nh_flags;
        }
out:
        return err;
}

static struct nexthop *nexthop_create(struct net *net, struct nh_config *cfg,
                                      struct netlink_ext_ack *extack)
{
        struct nh_info *nhi;
        struct nexthop *nh;
        int err = 0;

        nh = nexthop_alloc();
        if (!nh)
                return ERR_PTR(-ENOMEM);

        nhi = kzalloc_obj(*nhi);
        if (!nhi) {
                kfree(nh);
                return ERR_PTR(-ENOMEM);
        }

        nh->nh_flags = cfg->nh_flags;
        nh->net = net;

        nhi->nh_parent = nh;
        nhi->family = cfg->nh_family;
        nhi->fib_nhc.nhc_scope = RT_SCOPE_LINK;

        if (cfg->nh_fdb) {
                nhi->fdb_nh = 1;
                nhi->dst_port = cfg->nh_dst_port;
        }

        if (cfg->nh_blackhole) {
                nhi->reject_nh = 1;
                cfg->nh_ifindex = net->loopback_dev->ifindex;
        }

        switch (cfg->nh_family) {
        case AF_INET:
                err = nh_create_ipv4(net, nh, nhi, cfg, extack);
                break;
        case AF_INET6:
                err = nh_create_ipv6(net, nh, nhi, cfg, extack);
                break;
        }

        if (err) {
                kfree(nhi);
                kfree(nh);
                return ERR_PTR(err);
        }

        /* add the entry to the device based hash */
        if (!nhi->fdb_nh)
                nexthop_devhash_add(net, nhi);

        rcu_assign_pointer(nh->nh_info, nhi);

        return nh;
}

/* called with rtnl lock held */
static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg,
                                   struct netlink_ext_ack *extack)
{
        struct nexthop *nh;
        int err;

        if (!cfg->nh_id) {
                cfg->nh_id = nh_find_unused_id(net);
                if (!cfg->nh_id) {
                        NL_SET_ERR_MSG(extack, "No unused id");
                        return ERR_PTR(-EINVAL);
                }
        }

        if (cfg->nh_grp)
                nh = nexthop_create_group(net, cfg);
        else
                nh = nexthop_create(net, cfg, extack);

        if (IS_ERR(nh))
                return nh;

        refcount_set(&nh->refcnt, 1);
        nh->id = cfg->nh_id;
        nh->protocol = cfg->nh_protocol;
        nh->net = net;

        err = insert_nexthop(net, nh, cfg, extack);
        if (err) {
                WARN_ON_ONCE(__remove_nexthop(net, nh, NULL));
                nexthop_put(nh);
                nh = ERR_PTR(err);
        }

        return nh;
}

static int rtm_nh_get_timer(struct nlattr *attr, unsigned long fallback,
                            unsigned long *timer_p, bool *has_p,
                            struct netlink_ext_ack *extack)
{
        unsigned long timer;
        u32 value;

        if (!attr) {
                *timer_p = fallback;
                *has_p = false;
                return 0;
        }

        value = nla_get_u32(attr);
        timer = clock_t_to_jiffies(value);
        if (timer == ~0UL) {
                NL_SET_ERR_MSG(extack, "Timer value too large");
                return -EINVAL;
        }

        *timer_p = timer;
        *has_p = true;
        return 0;
}

static int rtm_to_nh_config_grp_res(struct nlattr *res, struct nh_config *cfg,
                                    struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_res_policy_new)] = {};
        int err;

        if (res) {
                err = nla_parse_nested(tb,
                                       ARRAY_SIZE(rtm_nh_res_policy_new) - 1,
                                       res, rtm_nh_res_policy_new, extack);
                if (err < 0)
                        return err;
        }

        if (tb[NHA_RES_GROUP_BUCKETS]) {
                cfg->nh_grp_res_num_buckets =
                        nla_get_u16(tb[NHA_RES_GROUP_BUCKETS]);
                cfg->nh_grp_res_has_num_buckets = true;
                if (!cfg->nh_grp_res_num_buckets) {
                        NL_SET_ERR_MSG(extack, "Number of buckets needs to be non-0");
                        return -EINVAL;
                }
        }

        err = rtm_nh_get_timer(tb[NHA_RES_GROUP_IDLE_TIMER],
                               NH_RES_DEFAULT_IDLE_TIMER,
                               &cfg->nh_grp_res_idle_timer,
                               &cfg->nh_grp_res_has_idle_timer,
                               extack);
        if (err)
                return err;

        return rtm_nh_get_timer(tb[NHA_RES_GROUP_UNBALANCED_TIMER],
                                NH_RES_DEFAULT_UNBALANCED_TIMER,
                                &cfg->nh_grp_res_unbalanced_timer,
                                &cfg->nh_grp_res_has_unbalanced_timer,
                                extack);
}

static int rtm_to_nh_config(struct net *net, struct sk_buff *skb,
                            struct nlmsghdr *nlh, struct nlattr **tb,
                            struct nh_config *cfg,
                            struct netlink_ext_ack *extack)
{
        struct nhmsg *nhm = nlmsg_data(nlh);
        int err;

        err = -EINVAL;
        if (nhm->resvd || nhm->nh_scope) {
                NL_SET_ERR_MSG(extack, "Invalid values in ancillary header");
                goto out;
        }
        if (nhm->nh_flags & ~NEXTHOP_VALID_USER_FLAGS) {
                NL_SET_ERR_MSG(extack, "Invalid nexthop flags in ancillary header");
                goto out;
        }

        switch (nhm->nh_family) {
        case AF_INET:
        case AF_INET6:
                break;
        case AF_UNSPEC:
                if (tb[NHA_GROUP])
                        break;
                fallthrough;
        default:
                NL_SET_ERR_MSG(extack, "Invalid address family");
                goto out;
        }

        memset(cfg, 0, sizeof(*cfg));
        cfg->nlflags = nlh->nlmsg_flags;
        cfg->nlinfo.portid = NETLINK_CB(skb).portid;
        cfg->nlinfo.nlh = nlh;
        cfg->nlinfo.nl_net = net;

        cfg->nh_family = nhm->nh_family;
        cfg->nh_protocol = nhm->nh_protocol;
        cfg->nh_flags = nhm->nh_flags;

        if (tb[NHA_ID])
                cfg->nh_id = nla_get_u32(tb[NHA_ID]);

        if (tb[NHA_FDB]) {
                if (tb[NHA_OIF] || tb[NHA_BLACKHOLE] ||
                    tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE]) {
                        NL_SET_ERR_MSG(extack, "Fdb attribute can not be used with encap, oif or blackhole");
                        goto out;
                }
                if (nhm->nh_flags) {
                        NL_SET_ERR_MSG(extack, "Unsupported nexthop flags in ancillary header");
                        goto out;
                }
                cfg->nh_fdb = nla_get_flag(tb[NHA_FDB]);
        }

        if (tb[NHA_DST_PORT]) {
                if (!tb[NHA_FDB] || !tb[NHA_GATEWAY]) {
                        NL_SET_ERR_MSG(extack,
                                       "Destination port can only be set on fdb nexthops that have a gateway");
                        goto out;
                }
                cfg->nh_dst_port = nla_get_be16(tb[NHA_DST_PORT]);
        }

        if (tb[NHA_GROUP]) {
                if (nhm->nh_family != AF_UNSPEC) {
                        NL_SET_ERR_MSG(extack, "Invalid family for group");
                        goto out;
                }
                cfg->nh_grp = tb[NHA_GROUP];

                cfg->nh_grp_type = NEXTHOP_GRP_TYPE_MPATH;
                if (tb[NHA_GROUP_TYPE])
                        cfg->nh_grp_type = nla_get_u16(tb[NHA_GROUP_TYPE]);

                if (cfg->nh_grp_type > NEXTHOP_GRP_TYPE_MAX) {
                        NL_SET_ERR_MSG(extack, "Invalid group type");
                        goto out;
                }

                err = nh_check_attr_group(net, tb, ARRAY_SIZE(rtm_nh_policy_new),
                                          cfg->nh_grp_type, extack);
                if (err)
                        goto out;

                if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_RES)
                        err = rtm_to_nh_config_grp_res(tb[NHA_RES_GROUP],
                                                       cfg, extack);

                if (tb[NHA_HW_STATS_ENABLE])
                        cfg->nh_hw_stats = nla_get_u32(tb[NHA_HW_STATS_ENABLE]);

                /* no other attributes should be set */
                goto out;
        }

        if (tb[NHA_BLACKHOLE]) {
                if (tb[NHA_GATEWAY] || tb[NHA_OIF] ||
                    tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE] || tb[NHA_FDB]) {
                        NL_SET_ERR_MSG(extack, "Blackhole attribute can not be used with gateway, oif, encap or fdb");
                        goto out;
                }

                cfg->nh_blackhole = 1;
                err = 0;
                goto out;
        }

        if (!cfg->nh_fdb && !tb[NHA_OIF]) {
                NL_SET_ERR_MSG(extack, "Device attribute required for non-blackhole and non-fdb nexthops");
                goto out;
        }

        err = -EINVAL;
        if (tb[NHA_GATEWAY]) {
                struct nlattr *gwa = tb[NHA_GATEWAY];

                switch (cfg->nh_family) {
                case AF_INET:
                        if (nla_len(gwa) != sizeof(u32)) {
                                NL_SET_ERR_MSG(extack, "Invalid gateway");
                                goto out;
                        }
                        cfg->gw.ipv4 = nla_get_be32(gwa);
                        break;
                case AF_INET6:
                        if (nla_len(gwa) != sizeof(struct in6_addr)) {
                                NL_SET_ERR_MSG(extack, "Invalid gateway");
                                goto out;
                        }
                        cfg->gw.ipv6 = nla_get_in6_addr(gwa);
                        break;
                default:
                        NL_SET_ERR_MSG(extack,
                                       "Unknown address family for gateway");
                        goto out;
                }
        } else {
                /* device only nexthop (no gateway) */
                if (cfg->nh_flags & RTNH_F_ONLINK) {
                        NL_SET_ERR_MSG(extack,
                                       "ONLINK flag can not be set for nexthop without a gateway");
                        goto out;
                }
        }

        if (tb[NHA_ENCAP]) {
                cfg->nh_encap = tb[NHA_ENCAP];

                if (!tb[NHA_ENCAP_TYPE]) {
                        NL_SET_ERR_MSG(extack, "LWT encapsulation type is missing");
                        goto out;
                }

                cfg->nh_encap_type = nla_get_u16(tb[NHA_ENCAP_TYPE]);
                err = lwtunnel_valid_encap_type(cfg->nh_encap_type, extack);
                if (err < 0)
                        goto out;

        } else if (tb[NHA_ENCAP_TYPE]) {
                NL_SET_ERR_MSG(extack, "LWT encapsulation attribute is missing");
                goto out;
        }

        if (tb[NHA_HW_STATS_ENABLE]) {
                NL_SET_ERR_MSG(extack, "Cannot enable nexthop hardware statistics for non-group nexthops");
                goto out;
        }

        err = 0;
out:
        return err;
}

static int rtm_to_nh_config_rtnl(struct net *net, struct nlattr **tb,
                                 struct nh_config *cfg,
                                 struct netlink_ext_ack *extack)
{
        if (tb[NHA_GROUP])
                return nh_check_attr_group_rtnl(net, tb, extack);

        if (tb[NHA_OIF]) {
                cfg->nh_ifindex = nla_get_u32(tb[NHA_OIF]);
                if (cfg->nh_ifindex)
                        cfg->dev = __dev_get_by_index(net, cfg->nh_ifindex);

                if (!cfg->dev) {
                        NL_SET_ERR_MSG(extack, "Invalid device index");
                        return -EINVAL;
                }

                if (!(cfg->dev->flags & IFF_UP)) {
                        NL_SET_ERR_MSG(extack, "Nexthop device is not up");
                        return -ENETDOWN;
                }

                if (!netif_carrier_ok(cfg->dev)) {
                        NL_SET_ERR_MSG(extack, "Carrier for nexthop device is down");
                        return -ENETDOWN;
                }
        }

        return 0;
}

/* rtnl */
static int rtm_new_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
                           struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_new)];
        struct net *net = sock_net(skb->sk);
        struct nh_config cfg;
        struct nexthop *nh;
        int err;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_new) - 1,
                          rtm_nh_policy_new, extack);
        if (err < 0)
                goto out;

        err = rtm_to_nh_config(net, skb, nlh, tb, &cfg, extack);
        if (err)
                goto out;

        if (cfg.nlflags & NLM_F_REPLACE && !cfg.nh_id) {
                NL_SET_ERR_MSG(extack, "Replace requires nexthop id");
                err = -EINVAL;
                goto out;
        }

        rtnl_net_lock(net);

        err = rtm_to_nh_config_rtnl(net, tb, &cfg, extack);
        if (err)
                goto unlock;

        nh = nexthop_add(net, &cfg, extack);
        if (IS_ERR(nh))
                err = PTR_ERR(nh);

unlock:
        rtnl_net_unlock(net);
out:
        return err;
}

static int nh_valid_get_del_req(const struct nlmsghdr *nlh,
                                struct nlattr **tb, u32 *id, u32 *op_flags,
                                struct netlink_ext_ack *extack)
{
        struct nhmsg *nhm = nlmsg_data(nlh);

        if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
                NL_SET_ERR_MSG(extack, "Invalid values in header");
                return -EINVAL;
        }

        if (!tb[NHA_ID]) {
                NL_SET_ERR_MSG(extack, "Nexthop id is missing");
                return -EINVAL;
        }

        *id = nla_get_u32(tb[NHA_ID]);
        if (!(*id)) {
                NL_SET_ERR_MSG(extack, "Invalid nexthop id");
                return -EINVAL;
        }

        if (op_flags)
                *op_flags = nla_get_u32_default(tb[NHA_OP_FLAGS], 0);

        return 0;
}

/* rtnl */
static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
                           struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_del)];
        struct net *net = sock_net(skb->sk);
        struct nl_info nlinfo = {
                .nlh = nlh,
                .nl_net = net,
                .portid = NETLINK_CB(skb).portid,
        };
        struct nexthop *nh;
        int err;
        u32 id;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_del) - 1, rtm_nh_policy_del,
                          extack);
        if (err < 0)
                return err;

        err = nh_valid_get_del_req(nlh, tb, &id, NULL, extack);
        if (err)
                return err;

        rtnl_net_lock(net);

        nh = nexthop_find_by_id(net, id);
        if (nh)
                remove_one_nexthop(net, nh, &nlinfo);
        else
                err = -ENOENT;

        rtnl_net_unlock(net);

        return err;
}

/* rtnl */
static int rtm_get_nexthop(struct sk_buff *in_skb, struct nlmsghdr *nlh,
                           struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_get)];
        struct net *net = sock_net(in_skb->sk);
        struct sk_buff *skb = NULL;
        struct nexthop *nh;
        u32 op_flags;
        int err;
        u32 id;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_get) - 1, rtm_nh_policy_get,
                          extack);
        if (err < 0)
                return err;

        err = nh_valid_get_del_req(nlh, tb, &id, &op_flags, extack);
        if (err)
                return err;

        err = -ENOENT;
        nh = nexthop_find_by_id(net, id);
        if (!nh)
                goto out;

        err = -ENOBUFS;
        skb = nlmsg_new(nh_nlmsg_size(nh, op_flags), GFP_KERNEL);
        if (!skb)
                goto out;

        err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP, NETLINK_CB(in_skb).portid,
                           nlh->nlmsg_seq, 0, op_flags);
        if (err < 0) {
                WARN_ON(err == -EMSGSIZE);
                goto errout_free;
        }

        err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
out:
        return err;
errout_free:
        kfree_skb(skb);
        goto out;
}

struct nh_dump_filter {
        u32 nh_id;
        int dev_idx;
        int master_idx;
        bool group_filter;
        bool fdb_filter;
        u32 res_bucket_nh_id;
        u32 op_flags;
};

static bool nh_dump_filtered(struct nexthop *nh,
                             struct nh_dump_filter *filter, u8 family)
{
        const struct net_device *dev;
        const struct nh_info *nhi;

        if (filter->group_filter && !nh->is_group)
                return true;

        if (!filter->dev_idx && !filter->master_idx && !family)
                return false;

        if (nh->is_group)
                return true;

        nhi = rtnl_dereference(nh->nh_info);
        if (family && nhi->family != family)
                return true;

        dev = nhi->fib_nhc.nhc_dev;
        if (filter->dev_idx && (!dev || dev->ifindex != filter->dev_idx))
                return true;

        if (filter->master_idx) {
                struct net_device *master;

                if (!dev)
                        return true;

                master = netdev_master_upper_dev_get((struct net_device *)dev);
                if (!master || master->ifindex != filter->master_idx)
                        return true;
        }

        return false;
}

static int __nh_valid_dump_req(const struct nlmsghdr *nlh, struct nlattr **tb,
                               struct nh_dump_filter *filter,
                               struct netlink_ext_ack *extack)
{
        struct nhmsg *nhm;
        u32 idx;

        if (tb[NHA_OIF]) {
                idx = nla_get_u32(tb[NHA_OIF]);
                if (idx > INT_MAX) {
                        NL_SET_ERR_MSG(extack, "Invalid device index");
                        return -EINVAL;
                }
                filter->dev_idx = idx;
        }
        if (tb[NHA_MASTER]) {
                idx = nla_get_u32(tb[NHA_MASTER]);
                if (idx > INT_MAX) {
                        NL_SET_ERR_MSG(extack, "Invalid master device index");
                        return -EINVAL;
                }
                filter->master_idx = idx;
        }
        filter->group_filter = nla_get_flag(tb[NHA_GROUPS]);
        filter->fdb_filter = nla_get_flag(tb[NHA_FDB]);

        nhm = nlmsg_data(nlh);
        if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
                NL_SET_ERR_MSG(extack, "Invalid values in header for nexthop dump request");
                return -EINVAL;
        }

        return 0;
}

static int nh_valid_dump_req(const struct nlmsghdr *nlh,
                             struct nh_dump_filter *filter,
                             struct netlink_callback *cb)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_dump)];
        int err;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_dump) - 1,
                          rtm_nh_policy_dump, cb->extack);
        if (err < 0)
                return err;

        filter->op_flags = nla_get_u32_default(tb[NHA_OP_FLAGS], 0);

        return __nh_valid_dump_req(nlh, tb, filter, cb->extack);
}

struct rtm_dump_nh_ctx {
        u32 idx;
};

static struct rtm_dump_nh_ctx *
rtm_dump_nh_ctx(struct netlink_callback *cb)
{
        struct rtm_dump_nh_ctx *ctx = (void *)cb->ctx;

        BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
        return ctx;
}

static int rtm_dump_walk_nexthops(struct sk_buff *skb,
                                  struct netlink_callback *cb,
                                  struct rb_root *root,
                                  struct rtm_dump_nh_ctx *ctx,
                                  int (*nh_cb)(struct sk_buff *skb,
                                               struct netlink_callback *cb,
                                               struct nexthop *nh, void *data),
                                  void *data)
{
        struct rb_node *node;
        int s_idx;
        int err;

        s_idx = ctx->idx;

        /* If this is not the first invocation, ctx->idx will contain the id of
         * the last nexthop we processed. Instead of starting from the very
         * first element of the red/black tree again and linearly skipping the
         * (potentially large) set of nodes with an id smaller than s_idx, walk
         * the tree and find the left-most node whose id is >= s_idx.  This
         * provides an efficient O(log n) starting point for the dump
         * continuation.
         */
        if (s_idx != 0) {
                struct rb_node *tmp = root->rb_node;

                node = NULL;
                while (tmp) {
                        struct nexthop *nh;

                        nh = rb_entry(tmp, struct nexthop, rb_node);
                        if (nh->id < s_idx) {
                                tmp = tmp->rb_right;
                        } else {
                                /* Track current candidate and keep looking on
                                 * the left side to find the left-most
                                 * (smallest id) that is still >= s_idx.
                                 */
                                node = tmp;
                                tmp = tmp->rb_left;
                        }
                }
        } else {
                node = rb_first(root);
        }

        for (; node; node = rb_next(node)) {
                struct nexthop *nh;

                nh = rb_entry(node, struct nexthop, rb_node);

                ctx->idx = nh->id;
                err = nh_cb(skb, cb, nh, data);
                if (err)
                        return err;
        }

        return 0;
}

static int rtm_dump_nexthop_cb(struct sk_buff *skb, struct netlink_callback *cb,
                               struct nexthop *nh, void *data)
{
        struct nhmsg *nhm = nlmsg_data(cb->nlh);
        struct nh_dump_filter *filter = data;

        if (nh_dump_filtered(nh, filter, nhm->nh_family))
                return 0;

        return nh_fill_node(skb, nh, RTM_NEWNEXTHOP,
                            NETLINK_CB(cb->skb).portid,
                            cb->nlh->nlmsg_seq, NLM_F_MULTI, filter->op_flags);
}

/* rtnl */
static int rtm_dump_nexthop(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct rtm_dump_nh_ctx *ctx = rtm_dump_nh_ctx(cb);
        struct net *net = sock_net(skb->sk);
        struct rb_root *root = &net->nexthop.rb_root;
        struct nh_dump_filter filter = {};
        int err;

        err = nh_valid_dump_req(cb->nlh, &filter, cb);
        if (err < 0)
                return err;

        err = rtm_dump_walk_nexthops(skb, cb, root, ctx,
                                     &rtm_dump_nexthop_cb, &filter);

        cb->seq = net->nexthop.seq;
        nl_dump_check_consistent(cb, nlmsg_hdr(skb));
        return err;
}

static struct nexthop *
nexthop_find_group_resilient(struct net *net, u32 id,
                             struct netlink_ext_ack *extack)
{
        struct nh_group *nhg;
        struct nexthop *nh;

        nh = nexthop_find_by_id(net, id);
        if (!nh)
                return ERR_PTR(-ENOENT);

        if (!nh->is_group) {
                NL_SET_ERR_MSG(extack, "Not a nexthop group");
                return ERR_PTR(-EINVAL);
        }

        nhg = rtnl_dereference(nh->nh_grp);
        if (!nhg->resilient) {
                NL_SET_ERR_MSG(extack, "Nexthop group not of type resilient");
                return ERR_PTR(-EINVAL);
        }

        return nh;
}

static int nh_valid_dump_nhid(struct nlattr *attr, u32 *nh_id_p,
                              struct netlink_ext_ack *extack)
{
        u32 idx;

        if (attr) {
                idx = nla_get_u32(attr);
                if (!idx) {
                        NL_SET_ERR_MSG(extack, "Invalid nexthop id");
                        return -EINVAL;
                }
                *nh_id_p = idx;
        } else {
                *nh_id_p = 0;
        }

        return 0;
}

static int nh_valid_dump_bucket_req(const struct nlmsghdr *nlh,
                                    struct nh_dump_filter *filter,
                                    struct netlink_callback *cb)
{
        struct nlattr *res_tb[ARRAY_SIZE(rtm_nh_res_bucket_policy_dump)];
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_dump_bucket)];
        int err;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_dump_bucket) - 1,
                          rtm_nh_policy_dump_bucket, NULL);
        if (err < 0)
                return err;

        err = nh_valid_dump_nhid(tb[NHA_ID], &filter->nh_id, cb->extack);
        if (err)
                return err;

        if (tb[NHA_RES_BUCKET]) {
                size_t max = ARRAY_SIZE(rtm_nh_res_bucket_policy_dump) - 1;

                err = nla_parse_nested(res_tb, max,
                                       tb[NHA_RES_BUCKET],
                                       rtm_nh_res_bucket_policy_dump,
                                       cb->extack);
                if (err < 0)
                        return err;

                err = nh_valid_dump_nhid(res_tb[NHA_RES_BUCKET_NH_ID],
                                         &filter->res_bucket_nh_id,
                                         cb->extack);
                if (err)
                        return err;
        }

        return __nh_valid_dump_req(nlh, tb, filter, cb->extack);
}

struct rtm_dump_res_bucket_ctx {
        struct rtm_dump_nh_ctx nh;
        u16 bucket_index;
};

static struct rtm_dump_res_bucket_ctx *
rtm_dump_res_bucket_ctx(struct netlink_callback *cb)
{
        struct rtm_dump_res_bucket_ctx *ctx = (void *)cb->ctx;

        BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
        return ctx;
}

struct rtm_dump_nexthop_bucket_data {
        struct rtm_dump_res_bucket_ctx *ctx;
        struct nh_dump_filter filter;
};

static int rtm_dump_nexthop_bucket_nh(struct sk_buff *skb,
                                      struct netlink_callback *cb,
                                      struct nexthop *nh,
                                      struct rtm_dump_nexthop_bucket_data *dd)
{
        u32 portid = NETLINK_CB(cb->skb).portid;
        struct nhmsg *nhm = nlmsg_data(cb->nlh);
        struct nh_res_table *res_table;
        struct nh_group *nhg;
        u16 bucket_index;
        int err;

        nhg = rtnl_dereference(nh->nh_grp);
        res_table = rtnl_dereference(nhg->res_table);
        for (bucket_index = dd->ctx->bucket_index;
             bucket_index < res_table->num_nh_buckets;
             bucket_index++) {
                struct nh_res_bucket *bucket;
                struct nh_grp_entry *nhge;

                bucket = &res_table->nh_buckets[bucket_index];
                nhge = rtnl_dereference(bucket->nh_entry);
                if (nh_dump_filtered(nhge->nh, &dd->filter, nhm->nh_family))
                        continue;

                if (dd->filter.res_bucket_nh_id &&
                    dd->filter.res_bucket_nh_id != nhge->nh->id)
                        continue;

                dd->ctx->bucket_index = bucket_index;
                err = nh_fill_res_bucket(skb, nh, bucket, bucket_index,
                                         RTM_NEWNEXTHOPBUCKET, portid,
                                         cb->nlh->nlmsg_seq, NLM_F_MULTI,
                                         cb->extack);
                if (err)
                        return err;
        }

        dd->ctx->bucket_index = 0;

        return 0;
}

static int rtm_dump_nexthop_bucket_cb(struct sk_buff *skb,
                                      struct netlink_callback *cb,
                                      struct nexthop *nh, void *data)
{
        struct rtm_dump_nexthop_bucket_data *dd = data;
        struct nh_group *nhg;

        if (!nh->is_group)
                return 0;

        nhg = rtnl_dereference(nh->nh_grp);
        if (!nhg->resilient)
                return 0;

        return rtm_dump_nexthop_bucket_nh(skb, cb, nh, dd);
}

/* rtnl */
static int rtm_dump_nexthop_bucket(struct sk_buff *skb,
                                   struct netlink_callback *cb)
{
        struct rtm_dump_res_bucket_ctx *ctx = rtm_dump_res_bucket_ctx(cb);
        struct rtm_dump_nexthop_bucket_data dd = { .ctx = ctx };
        struct net *net = sock_net(skb->sk);
        struct nexthop *nh;
        int err;

        err = nh_valid_dump_bucket_req(cb->nlh, &dd.filter, cb);
        if (err)
                return err;

        if (dd.filter.nh_id) {
                nh = nexthop_find_group_resilient(net, dd.filter.nh_id,
                                                  cb->extack);
                if (IS_ERR(nh))
                        return PTR_ERR(nh);
                err = rtm_dump_nexthop_bucket_nh(skb, cb, nh, &dd);
        } else {
                struct rb_root *root = &net->nexthop.rb_root;

                err = rtm_dump_walk_nexthops(skb, cb, root, &ctx->nh,
                                             &rtm_dump_nexthop_bucket_cb, &dd);
        }

        cb->seq = net->nexthop.seq;
        nl_dump_check_consistent(cb, nlmsg_hdr(skb));
        return err;
}

static int nh_valid_get_bucket_req_res_bucket(struct nlattr *res,
                                              u16 *bucket_index,
                                              struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_res_bucket_policy_get)];
        int err;

        err = nla_parse_nested(tb, ARRAY_SIZE(rtm_nh_res_bucket_policy_get) - 1,
                               res, rtm_nh_res_bucket_policy_get, extack);
        if (err < 0)
                return err;

        if (!tb[NHA_RES_BUCKET_INDEX]) {
                NL_SET_ERR_MSG(extack, "Bucket index is missing");
                return -EINVAL;
        }

        *bucket_index = nla_get_u16(tb[NHA_RES_BUCKET_INDEX]);
        return 0;
}

static int nh_valid_get_bucket_req(const struct nlmsghdr *nlh,
                                   u32 *id, u16 *bucket_index,
                                   struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_get_bucket)];
        int err;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_get_bucket) - 1,
                          rtm_nh_policy_get_bucket, extack);
        if (err < 0)
                return err;

        err = nh_valid_get_del_req(nlh, tb, id, NULL, extack);
        if (err)
                return err;

        if (!tb[NHA_RES_BUCKET]) {
                NL_SET_ERR_MSG(extack, "Bucket information is missing");
                return -EINVAL;
        }

        err = nh_valid_get_bucket_req_res_bucket(tb[NHA_RES_BUCKET],
                                                 bucket_index, extack);
        if (err)
                return err;

        return 0;
}

/* rtnl */
static int rtm_get_nexthop_bucket(struct sk_buff *in_skb, struct nlmsghdr *nlh,
                                  struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        struct nh_res_table *res_table;
        struct sk_buff *skb = NULL;
        struct nh_group *nhg;
        struct nexthop *nh;
        u16 bucket_index;
        int err;
        u32 id;

        err = nh_valid_get_bucket_req(nlh, &id, &bucket_index, extack);
        if (err)
                return err;

        nh = nexthop_find_group_resilient(net, id, extack);
        if (IS_ERR(nh))
                return PTR_ERR(nh);

        nhg = rtnl_dereference(nh->nh_grp);
        res_table = rtnl_dereference(nhg->res_table);
        if (bucket_index >= res_table->num_nh_buckets) {
                NL_SET_ERR_MSG(extack, "Bucket index out of bounds");
                return -ENOENT;
        }

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb)
                return -ENOBUFS;

        err = nh_fill_res_bucket(skb, nh, &res_table->nh_buckets[bucket_index],
                                 bucket_index, RTM_NEWNEXTHOPBUCKET,
                                 NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
                                 0, extack);
        if (err < 0) {
                WARN_ON(err == -EMSGSIZE);
                goto errout_free;
        }

        return rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);

errout_free:
        kfree_skb(skb);
        return err;
}

static void nexthop_sync_mtu(struct net_device *dev, u32 orig_mtu)
{
        unsigned int hash = nh_dev_hashfn(dev->ifindex);
        struct net *net = dev_net(dev);
        struct hlist_head *head = &net->nexthop.devhash[hash];
        struct hlist_node *n;
        struct nh_info *nhi;

        hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
                if (nhi->fib_nhc.nhc_dev == dev) {
                        if (nhi->family == AF_INET)
                                fib_nhc_update_mtu(&nhi->fib_nhc, dev->mtu,
                                                   orig_mtu);
                }
        }
}

/* rtnl */
static int nh_netdev_event(struct notifier_block *this,
                           unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct netdev_notifier_info_ext *info_ext;

        switch (event) {
        case NETDEV_DOWN:
        case NETDEV_UNREGISTER:
                nexthop_flush_dev(dev, event);
                break;
        case NETDEV_CHANGE:
                if (!(netif_get_flags(dev) & (IFF_RUNNING | IFF_LOWER_UP)))
                        nexthop_flush_dev(dev, event);
                break;
        case NETDEV_CHANGEMTU:
                info_ext = ptr;
                nexthop_sync_mtu(dev, info_ext->ext.mtu);
                rt_cache_flush(dev_net(dev));
                break;
        }
        return NOTIFY_DONE;
}

static struct notifier_block nh_netdev_notifier = {
        .notifier_call = nh_netdev_event,
};

static int nexthops_dump(struct net *net, struct notifier_block *nb,
                         enum nexthop_event_type event_type,
                         struct netlink_ext_ack *extack)
{
        struct rb_root *root = &net->nexthop.rb_root;
        struct rb_node *node;
        int err = 0;

        for (node = rb_first(root); node; node = rb_next(node)) {
                struct nexthop *nh;

                nh = rb_entry(node, struct nexthop, rb_node);
                err = call_nexthop_notifier(nb, net, event_type, nh, extack);
                if (err)
                        break;
        }

        return err;
}

int register_nexthop_notifier(struct net *net, struct notifier_block *nb,
                              struct netlink_ext_ack *extack)
{
        int err;

        rtnl_lock();
        err = nexthops_dump(net, nb, NEXTHOP_EVENT_REPLACE, extack);
        if (err)
                goto unlock;
        err = blocking_notifier_chain_register(&net->nexthop.notifier_chain,
                                               nb);
unlock:
        rtnl_unlock();
        return err;
}
EXPORT_SYMBOL(register_nexthop_notifier);

int __unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
{
        int err;

        err = blocking_notifier_chain_unregister(&net->nexthop.notifier_chain,
                                                 nb);
        if (!err)
                nexthops_dump(net, nb, NEXTHOP_EVENT_DEL, NULL);
        return err;
}
EXPORT_SYMBOL(__unregister_nexthop_notifier);

int unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
{
        int err;

        rtnl_lock();
        err = __unregister_nexthop_notifier(net, nb);
        rtnl_unlock();
        return err;
}
EXPORT_SYMBOL(unregister_nexthop_notifier);

void nexthop_set_hw_flags(struct net *net, u32 id, bool offload, bool trap)
{
        struct nexthop *nexthop;

        rcu_read_lock();

        nexthop = nexthop_find_by_id(net, id);
        if (!nexthop)
                goto out;

        nexthop->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
        if (offload)
                nexthop->nh_flags |= RTNH_F_OFFLOAD;
        if (trap)
                nexthop->nh_flags |= RTNH_F_TRAP;

out:
        rcu_read_unlock();
}
EXPORT_SYMBOL(nexthop_set_hw_flags);

void nexthop_bucket_set_hw_flags(struct net *net, u32 id, u16 bucket_index,
                                 bool offload, bool trap)
{
        struct nh_res_table *res_table;
        struct nh_res_bucket *bucket;
        struct nexthop *nexthop;
        struct nh_group *nhg;

        rcu_read_lock();

        nexthop = nexthop_find_by_id(net, id);
        if (!nexthop || !nexthop->is_group)
                goto out;

        nhg = rcu_dereference(nexthop->nh_grp);
        if (!nhg->resilient)
                goto out;

        if (bucket_index >= nhg->res_table->num_nh_buckets)
                goto out;

        res_table = rcu_dereference(nhg->res_table);
        bucket = &res_table->nh_buckets[bucket_index];
        bucket->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
        if (offload)
                bucket->nh_flags |= RTNH_F_OFFLOAD;
        if (trap)
                bucket->nh_flags |= RTNH_F_TRAP;

out:
        rcu_read_unlock();
}
EXPORT_SYMBOL(nexthop_bucket_set_hw_flags);

void nexthop_res_grp_activity_update(struct net *net, u32 id, u16 num_buckets,
                                     unsigned long *activity)
{
        struct nh_res_table *res_table;
        struct nexthop *nexthop;
        struct nh_group *nhg;
        u16 i;

        rcu_read_lock();

        nexthop = nexthop_find_by_id(net, id);
        if (!nexthop || !nexthop->is_group)
                goto out;

        nhg = rcu_dereference(nexthop->nh_grp);
        if (!nhg->resilient)
                goto out;

        /* Instead of silently ignoring some buckets, demand that the sizes
         * be the same.
         */
        res_table = rcu_dereference(nhg->res_table);
        if (num_buckets != res_table->num_nh_buckets)
                goto out;

        for (i = 0; i < num_buckets; i++) {
                if (test_bit(i, activity))
                        nh_res_bucket_set_busy(&res_table->nh_buckets[i]);
        }

out:
        rcu_read_unlock();
}
EXPORT_SYMBOL(nexthop_res_grp_activity_update);

static void __net_exit nexthop_net_exit_rtnl(struct net *net,
                                             struct list_head *dev_to_kill)
{
        ASSERT_RTNL_NET(net);
        flush_all_nexthops(net);
}

static void __net_exit nexthop_net_exit(struct net *net)
{
        kfree(net->nexthop.devhash);
        net->nexthop.devhash = NULL;
}

static int __net_init nexthop_net_init(struct net *net)
{
        size_t sz = sizeof(struct hlist_head) * NH_DEV_HASHSIZE;

        net->nexthop.rb_root = RB_ROOT;
        net->nexthop.devhash = kzalloc(sz, GFP_KERNEL);
        if (!net->nexthop.devhash)
                return -ENOMEM;
        BLOCKING_INIT_NOTIFIER_HEAD(&net->nexthop.notifier_chain);

        return 0;
}

static struct pernet_operations nexthop_net_ops = {
        .init = nexthop_net_init,
        .exit = nexthop_net_exit,
        .exit_rtnl = nexthop_net_exit_rtnl,
};

static const struct rtnl_msg_handler nexthop_rtnl_msg_handlers[] __initconst = {
        {.msgtype = RTM_NEWNEXTHOP, .doit = rtm_new_nexthop,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.msgtype = RTM_DELNEXTHOP, .doit = rtm_del_nexthop,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.msgtype = RTM_GETNEXTHOP, .doit = rtm_get_nexthop,
         .dumpit = rtm_dump_nexthop},
        {.msgtype = RTM_GETNEXTHOPBUCKET, .doit = rtm_get_nexthop_bucket,
         .dumpit = rtm_dump_nexthop_bucket},
        {.protocol = PF_INET, .msgtype = RTM_NEWNEXTHOP,
         .doit = rtm_new_nexthop, .flags = RTNL_FLAG_DOIT_PERNET},
        {.protocol = PF_INET, .msgtype = RTM_GETNEXTHOP,
         .dumpit = rtm_dump_nexthop},
        {.protocol = PF_INET6, .msgtype = RTM_NEWNEXTHOP,
         .doit = rtm_new_nexthop, .flags = RTNL_FLAG_DOIT_PERNET},
        {.protocol = PF_INET6, .msgtype = RTM_GETNEXTHOP,
         .dumpit = rtm_dump_nexthop},
};

static int __init nexthop_init(void)
{
        int err;

        err = register_pernet_subsys(&nexthop_net_ops);
        if (err)
                return err;

        err = register_netdevice_notifier(&nh_netdev_notifier);
        if (err)
                goto err_unregister_pernet;

        rtnl_register_many(nexthop_rtnl_msg_handlers);

        return 0;

err_unregister_pernet:
        unregister_pernet_subsys(&nexthop_net_ops);
        return err;
}
subsys_initcall(nexthop_init);





























    3 


























   13 





   13 
   13 





   13 





   13 








    4 
    4 
    4 




















    4 


    3 
    3 



    5 





    8 










   15 



    1 



   13 
    1 




















   46 













    4 



    1 
    7 



    1 









    1 





   33 






    1 




   11 







    1 



    5 

    4 





    2 
    7 
    4 







    1 



    9 

    1 




    4 

    5 







    5 
    1 





    1 





    2 











   18 

   18 
   18 
   18 

   18 





   18 


   18 

   18 












   16 




   15 
    1 




    5 










    5 
    5 






    5 



    5 



    5 


    5 











    5 












   18 





    5 







    1 



    5 






    3 












    4 

    4 


    1 


    3 























   25 


   25 































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (C) 2019 Netronome Systems, Inc. */

#include <linux/if_arp.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mpls.h>
#include <linux/rtnetlink.h>
#include <linux/skbuff.h>
#include <linux/tc_act/tc_mpls.h>
#include <net/mpls.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/tc_act/tc_mpls.h>
#include <net/tc_wrapper.h>

static struct tc_action_ops act_mpls_ops;

#define ACT_MPLS_TTL_DEFAULT        255

static __be32 tcf_mpls_get_lse(struct mpls_shim_hdr *lse,
                               struct tcf_mpls_params *p, bool set_bos)
{
        u32 new_lse = 0;

        if (lse)
                new_lse = be32_to_cpu(lse->label_stack_entry);

        if (p->tcfm_label != ACT_MPLS_LABEL_NOT_SET) {
                new_lse &= ~MPLS_LS_LABEL_MASK;
                new_lse |= p->tcfm_label << MPLS_LS_LABEL_SHIFT;
        }
        if (p->tcfm_ttl) {
                new_lse &= ~MPLS_LS_TTL_MASK;
                new_lse |= p->tcfm_ttl << MPLS_LS_TTL_SHIFT;
        }
        if (p->tcfm_tc != ACT_MPLS_TC_NOT_SET) {
                new_lse &= ~MPLS_LS_TC_MASK;
                new_lse |= p->tcfm_tc << MPLS_LS_TC_SHIFT;
        }
        if (p->tcfm_bos != ACT_MPLS_BOS_NOT_SET) {
                new_lse &= ~MPLS_LS_S_MASK;
                new_lse |= p->tcfm_bos << MPLS_LS_S_SHIFT;
        } else if (set_bos) {
                new_lse |= 1 << MPLS_LS_S_SHIFT;
        }

        return cpu_to_be32(new_lse);
}

TC_INDIRECT_SCOPE int tcf_mpls_act(struct sk_buff *skb,
                                   const struct tc_action *a,
                                   struct tcf_result *res)
{
        struct tcf_mpls *m = to_mpls(a);
        struct tcf_mpls_params *p;
        __be32 new_lse;
        int mac_len;

        tcf_lastuse_update(&m->tcf_tm);
        bstats_update(this_cpu_ptr(m->common.cpu_bstats), skb);

        /* Ensure 'data' points at mac_header prior calling mpls manipulating
         * functions.
         */
        if (skb_at_tc_ingress(skb)) {
                skb_push_rcsum(skb, skb->mac_len);
                mac_len = skb->mac_len;
        } else {
                mac_len = skb_network_offset(skb);
        }

        p = rcu_dereference_bh(m->mpls_p);

        switch (p->tcfm_action) {
        case TCA_MPLS_ACT_POP:
                if (skb_mpls_pop(skb, p->tcfm_proto, mac_len,
                                 skb->dev && skb->dev->type == ARPHRD_ETHER))
                        goto drop;
                break;
        case TCA_MPLS_ACT_PUSH:
                new_lse = tcf_mpls_get_lse(NULL, p, !eth_p_mpls(skb_protocol(skb, true)));
                if (skb_mpls_push(skb, new_lse, p->tcfm_proto, mac_len,
                                  skb->dev && skb->dev->type == ARPHRD_ETHER))
                        goto drop;
                break;
        case TCA_MPLS_ACT_MAC_PUSH:
                if (skb_vlan_tag_present(skb)) {
                        if (__vlan_insert_inner_tag(skb, skb->vlan_proto,
                                                    skb_vlan_tag_get(skb),
                                                    ETH_HLEN) < 0)
                                goto drop;

                        skb->protocol = skb->vlan_proto;
                        __vlan_hwaccel_clear_tag(skb);
                }

                new_lse = tcf_mpls_get_lse(NULL, p, mac_len ||
                                           !eth_p_mpls(skb->protocol));

                if (skb_mpls_push(skb, new_lse, p->tcfm_proto, 0, false))
                        goto drop;
                break;
        case TCA_MPLS_ACT_MODIFY:
                if (!pskb_may_pull(skb,
                                   skb_network_offset(skb) + MPLS_HLEN))
                        goto drop;
                new_lse = tcf_mpls_get_lse(mpls_hdr(skb), p, false);
                if (skb_mpls_update_lse(skb, new_lse))
                        goto drop;
                break;
        case TCA_MPLS_ACT_DEC_TTL:
                if (skb_mpls_dec_ttl(skb))
                        goto drop;
                break;
        }

        if (skb_at_tc_ingress(skb))
                skb_pull_rcsum(skb, skb->mac_len);

        return p->action;

drop:
        qstats_cpu_drop_inc(m->common.cpu_qstats);
        return TC_ACT_SHOT;
}

static int valid_label(const struct nlattr *attr,
                       struct netlink_ext_ack *extack)
{
        const u32 *label = nla_data(attr);

        if (nla_len(attr) != sizeof(*label)) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid MPLS label length");
                return -EINVAL;
        }

        if (*label & ~MPLS_LABEL_MASK || *label == MPLS_LABEL_IMPLNULL) {
                NL_SET_ERR_MSG_MOD(extack, "MPLS label out of range");
                return -EINVAL;
        }

        return 0;
}

static const struct nla_policy mpls_policy[TCA_MPLS_MAX + 1] = {
        [TCA_MPLS_PARMS]        = NLA_POLICY_EXACT_LEN(sizeof(struct tc_mpls)),
        [TCA_MPLS_PROTO]        = { .type = NLA_U16 },
        [TCA_MPLS_LABEL]        = NLA_POLICY_VALIDATE_FN(NLA_BINARY,
                                                         valid_label),
        [TCA_MPLS_TC]                = NLA_POLICY_RANGE(NLA_U8, 0, 7),
        [TCA_MPLS_TTL]                = NLA_POLICY_MIN(NLA_U8, 1),
        [TCA_MPLS_BOS]                = NLA_POLICY_RANGE(NLA_U8, 0, 1),
};

static int tcf_mpls_init(struct net *net, struct nlattr *nla,
                         struct nlattr *est, struct tc_action **a,
                         struct tcf_proto *tp, u32 flags,
                         struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, act_mpls_ops.net_id);
        bool bind = flags & TCA_ACT_FLAGS_BIND;
        struct nlattr *tb[TCA_MPLS_MAX + 1];
        struct tcf_chain *goto_ch = NULL;
        struct tcf_mpls_params *p;
        struct tc_mpls *parm;
        bool exists = false;
        struct tcf_mpls *m;
        int ret = 0, err;
        u8 mpls_ttl = 0;
        u32 index;

        if (!nla) {
                NL_SET_ERR_MSG_MOD(extack, "Missing netlink attributes");
                return -EINVAL;
        }

        err = nla_parse_nested(tb, TCA_MPLS_MAX, nla, mpls_policy, extack);
        if (err < 0)
                return err;

        if (!tb[TCA_MPLS_PARMS]) {
                NL_SET_ERR_MSG_MOD(extack, "No MPLS params");
                return -EINVAL;
        }
        parm = nla_data(tb[TCA_MPLS_PARMS]);
        index = parm->index;

        err = tcf_idr_check_alloc(tn, &index, a, bind);
        if (err < 0)
                return err;
        exists = err;
        if (exists && bind)
                return ACT_P_BOUND;

        if (!exists) {
                ret = tcf_idr_create(tn, index, est, a, &act_mpls_ops, bind,
                                     true, flags);
                if (ret) {
                        tcf_idr_cleanup(tn, index);
                        return ret;
                }

                ret = ACT_P_CREATED;
        } else if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
                tcf_idr_release(*a, bind);
                return -EEXIST;
        }

        /* Verify parameters against action type. */
        switch (parm->m_action) {
        case TCA_MPLS_ACT_POP:
                if (!tb[TCA_MPLS_PROTO]) {
                        NL_SET_ERR_MSG_MOD(extack, "Protocol must be set for MPLS pop");
                        err = -EINVAL;
                        goto release_idr;
                }
                if (!eth_proto_is_802_3(nla_get_be16(tb[TCA_MPLS_PROTO]))) {
                        NL_SET_ERR_MSG_MOD(extack, "Invalid protocol type for MPLS pop");
                        err = -EINVAL;
                        goto release_idr;
                }
                if (tb[TCA_MPLS_LABEL] || tb[TCA_MPLS_TTL] || tb[TCA_MPLS_TC] ||
                    tb[TCA_MPLS_BOS]) {
                        NL_SET_ERR_MSG_MOD(extack, "Label, TTL, TC or BOS cannot be used with MPLS pop");
                        err = -EINVAL;
                        goto release_idr;
                }
                break;
        case TCA_MPLS_ACT_DEC_TTL:
                if (tb[TCA_MPLS_PROTO] || tb[TCA_MPLS_LABEL] ||
                    tb[TCA_MPLS_TTL] || tb[TCA_MPLS_TC] || tb[TCA_MPLS_BOS]) {
                        NL_SET_ERR_MSG_MOD(extack, "Label, TTL, TC, BOS or protocol cannot be used with MPLS dec_ttl");
                        err = -EINVAL;
                        goto release_idr;
                }
                break;
        case TCA_MPLS_ACT_PUSH:
        case TCA_MPLS_ACT_MAC_PUSH:
                if (!tb[TCA_MPLS_LABEL]) {
                        NL_SET_ERR_MSG_MOD(extack, "Label is required for MPLS push");
                        err = -EINVAL;
                        goto release_idr;
                }
                if (tb[TCA_MPLS_PROTO] &&
                    !eth_p_mpls(nla_get_be16(tb[TCA_MPLS_PROTO]))) {
                        NL_SET_ERR_MSG_MOD(extack, "Protocol must be an MPLS type for MPLS push");
                        err = -EPROTONOSUPPORT;
                        goto release_idr;
                }
                /* Push needs a TTL - if not specified, set a default value. */
                if (!tb[TCA_MPLS_TTL]) {
#if IS_ENABLED(CONFIG_MPLS)
                        mpls_ttl = net->mpls.default_ttl ?
                                   net->mpls.default_ttl : ACT_MPLS_TTL_DEFAULT;
#else
                        mpls_ttl = ACT_MPLS_TTL_DEFAULT;
#endif
                }
                break;
        case TCA_MPLS_ACT_MODIFY:
                if (tb[TCA_MPLS_PROTO]) {
                        NL_SET_ERR_MSG_MOD(extack, "Protocol cannot be used with MPLS modify");
                        err = -EINVAL;
                        goto release_idr;
                }
                break;
        default:
                NL_SET_ERR_MSG_MOD(extack, "Unknown MPLS action");
                err = -EINVAL;
                goto release_idr;
        }

        err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
        if (err < 0)
                goto release_idr;

        m = to_mpls(*a);

        p = kzalloc_obj(*p);
        if (!p) {
                err = -ENOMEM;
                goto put_chain;
        }

        p->tcfm_action = parm->m_action;
        p->tcfm_label = nla_get_u32_default(tb[TCA_MPLS_LABEL],
                                            ACT_MPLS_LABEL_NOT_SET);
        p->tcfm_tc = nla_get_u8_default(tb[TCA_MPLS_TC], ACT_MPLS_TC_NOT_SET);
        p->tcfm_ttl = nla_get_u8_default(tb[TCA_MPLS_TTL], mpls_ttl);
        p->tcfm_bos = nla_get_u8_default(tb[TCA_MPLS_BOS],
                                         ACT_MPLS_BOS_NOT_SET);
        p->tcfm_proto = nla_get_be16_default(tb[TCA_MPLS_PROTO],
                                             htons(ETH_P_MPLS_UC));
        p->action = parm->action;

        spin_lock_bh(&m->tcf_lock);
        goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
        p = rcu_replace_pointer(m->mpls_p, p, lockdep_is_held(&m->tcf_lock));
        spin_unlock_bh(&m->tcf_lock);

        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
        if (p)
                kfree_rcu(p, rcu);

        return ret;
put_chain:
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
release_idr:
        tcf_idr_release(*a, bind);
        return err;
}

static void tcf_mpls_cleanup(struct tc_action *a)
{
        struct tcf_mpls *m = to_mpls(a);
        struct tcf_mpls_params *p;

        p = rcu_dereference_protected(m->mpls_p, 1);
        if (p)
                kfree_rcu(p, rcu);
}

static int tcf_mpls_dump(struct sk_buff *skb, struct tc_action *a,
                         int bind, int ref)
{
        unsigned char *b = skb_tail_pointer(skb);
        const struct tcf_mpls *m = to_mpls(a);
        const struct tcf_mpls_params *p;
        struct tc_mpls opt = {
                .index    = m->tcf_index,
                .refcnt   = refcount_read(&m->tcf_refcnt) - ref,
                .bindcnt  = atomic_read(&m->tcf_bindcnt) - bind,
        };
        struct tcf_t t;

        rcu_read_lock();
        p = rcu_dereference(m->mpls_p);
        opt.m_action = p->tcfm_action;
        opt.action = p->action;

        if (nla_put(skb, TCA_MPLS_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;

        if (p->tcfm_label != ACT_MPLS_LABEL_NOT_SET &&
            nla_put_u32(skb, TCA_MPLS_LABEL, p->tcfm_label))
                goto nla_put_failure;

        if (p->tcfm_tc != ACT_MPLS_TC_NOT_SET &&
            nla_put_u8(skb, TCA_MPLS_TC, p->tcfm_tc))
                goto nla_put_failure;

        if (p->tcfm_ttl && nla_put_u8(skb, TCA_MPLS_TTL, p->tcfm_ttl))
                goto nla_put_failure;

        if (p->tcfm_bos != ACT_MPLS_BOS_NOT_SET &&
            nla_put_u8(skb, TCA_MPLS_BOS, p->tcfm_bos))
                goto nla_put_failure;

        if (nla_put_be16(skb, TCA_MPLS_PROTO, p->tcfm_proto))
                goto nla_put_failure;

        tcf_tm_dump(&t, &m->tcf_tm);

        if (nla_put_64bit(skb, TCA_MPLS_TM, sizeof(t), &t, TCA_MPLS_PAD))
                goto nla_put_failure;

        rcu_read_unlock();

        return skb->len;

nla_put_failure:
        rcu_read_unlock();
        nlmsg_trim(skb, b);
        return -EMSGSIZE;
}

static int tcf_mpls_offload_act_setup(struct tc_action *act, void *entry_data,
                                      u32 *index_inc, bool bind,
                                      struct netlink_ext_ack *extack)
{
        if (bind) {
                struct flow_action_entry *entry = entry_data;

                switch (tcf_mpls_action(act)) {
                case TCA_MPLS_ACT_PUSH:
                        entry->id = FLOW_ACTION_MPLS_PUSH;
                        entry->mpls_push.proto = tcf_mpls_proto(act);
                        entry->mpls_push.label = tcf_mpls_label(act);
                        entry->mpls_push.tc = tcf_mpls_tc(act);
                        entry->mpls_push.bos = tcf_mpls_bos(act);
                        entry->mpls_push.ttl = tcf_mpls_ttl(act);
                        break;
                case TCA_MPLS_ACT_POP:
                        entry->id = FLOW_ACTION_MPLS_POP;
                        entry->mpls_pop.proto = tcf_mpls_proto(act);
                        break;
                case TCA_MPLS_ACT_MODIFY:
                        entry->id = FLOW_ACTION_MPLS_MANGLE;
                        entry->mpls_mangle.label = tcf_mpls_label(act);
                        entry->mpls_mangle.tc = tcf_mpls_tc(act);
                        entry->mpls_mangle.bos = tcf_mpls_bos(act);
                        entry->mpls_mangle.ttl = tcf_mpls_ttl(act);
                        break;
                case TCA_MPLS_ACT_DEC_TTL:
                        NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"dec_ttl\" option is used");
                        return -EOPNOTSUPP;
                case TCA_MPLS_ACT_MAC_PUSH:
                        NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"mac_push\" option is used");
                        return -EOPNOTSUPP;
                default:
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported MPLS mode offload");
                        return -EOPNOTSUPP;
                }
                *index_inc = 1;
        } else {
                struct flow_offload_action *fl_action = entry_data;

                switch (tcf_mpls_action(act)) {
                case TCA_MPLS_ACT_PUSH:
                        fl_action->id = FLOW_ACTION_MPLS_PUSH;
                        break;
                case TCA_MPLS_ACT_POP:
                        fl_action->id = FLOW_ACTION_MPLS_POP;
                        break;
                case TCA_MPLS_ACT_MODIFY:
                        fl_action->id = FLOW_ACTION_MPLS_MANGLE;
                        break;
                default:
                        return -EOPNOTSUPP;
                }
        }

        return 0;
}

static struct tc_action_ops act_mpls_ops = {
        .kind                =        "mpls",
        .id                =        TCA_ID_MPLS,
        .owner                =        THIS_MODULE,
        .act                =        tcf_mpls_act,
        .dump                =        tcf_mpls_dump,
        .init                =        tcf_mpls_init,
        .cleanup        =        tcf_mpls_cleanup,
        .offload_act_setup =        tcf_mpls_offload_act_setup,
        .size                =        sizeof(struct tcf_mpls),
};
MODULE_ALIAS_NET_ACT("mpls");

static __net_init int mpls_init_net(struct net *net)
{
        struct tc_action_net *tn = net_generic(net, act_mpls_ops.net_id);

        return tc_action_net_init(net, tn, &act_mpls_ops);
}

static void __net_exit mpls_exit_net(struct list_head *net_list)
{
        tc_action_net_exit(net_list, act_mpls_ops.net_id);
}

static struct pernet_operations mpls_net_ops = {
        .init = mpls_init_net,
        .exit_batch = mpls_exit_net,
        .id   = &act_mpls_ops.net_id,
        .size = sizeof(struct tc_action_net),
};

static int __init mpls_init_module(void)
{
        return tcf_register_action(&act_mpls_ops, &mpls_net_ops);
}

static void __exit mpls_cleanup_module(void)
{
        tcf_unregister_action(&act_mpls_ops, &mpls_net_ops);
}

module_init(mpls_init_module);
module_exit(mpls_cleanup_module);

MODULE_SOFTDEP("post: mpls_gso");
MODULE_AUTHOR("Netronome Systems <oss-drivers@netronome.com>");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("MPLS manipulation actions");
























































































































































    4 





   17 





   39 


   39 


   32 


   12 
   14 











   13 
   18 




   21 
   24 





   24 





   23 
    1 








   11 
   13 



    5 






    7 






    3 









    3 












   10 







    1 

    1 

    8 


















    4 







    6 







    3 














































    3 






    3 






    2 










    2 
































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library functions
 *
 * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
 * Copyright (c) 2003 Kyle McMartin <kyle@debian.org>
 * Copyright 2025 Google LLC
 */

#include <crypto/hmac.h>
#include <crypto/sha2.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/overflow.h>
#include <linux/string.h>
#include <linux/unaligned.h>
#include <linux/wordpart.h>
#include "fips-sha.h"

static const struct sha512_block_state sha384_iv = {
        .h = {
                SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3,
                SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7,
        },
};

static const struct sha512_block_state sha512_iv = {
        .h = {
                SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3,
                SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7,
        },
};

static const u64 sha512_K[80] = {
        0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL,
        0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL,
        0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL,
        0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL,
        0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL,
        0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL,
        0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL,
        0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL,
        0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL,
        0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL,
        0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL,
        0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL,
        0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL,
        0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL,
        0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL,
        0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL,
        0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL,
        0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL,
        0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL,
        0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL,
        0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL,
        0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL,
        0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL,
        0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL,
        0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL,
        0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL,
        0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL,
};

#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z))))
#define Maj(x, y, z) (((x) & (y)) | ((z) & ((x) | (y))))
#define e0(x) (ror64((x), 28) ^ ror64((x), 34) ^ ror64((x), 39))
#define e1(x) (ror64((x), 14) ^ ror64((x), 18) ^ ror64((x), 41))
#define s0(x) (ror64((x), 1) ^ ror64((x), 8) ^ ((x) >> 7))
#define s1(x) (ror64((x), 19) ^ ror64((x), 61) ^ ((x) >> 6))

static void sha512_block_generic(struct sha512_block_state *state,
                                 const u8 *data)
{
        u64 a = state->h[0];
        u64 b = state->h[1];
        u64 c = state->h[2];
        u64 d = state->h[3];
        u64 e = state->h[4];
        u64 f = state->h[5];
        u64 g = state->h[6];
        u64 h = state->h[7];
        u64 t1, t2;
        u64 W[16];

        for (int j = 0; j < 16; j++)
                W[j] = get_unaligned_be64(data + j * sizeof(u64));

        for (int i = 0; i < 80; i += 8) {
                if ((i & 15) == 0 && i != 0) {
                        for (int j = 0; j < 16; j++) {
                                W[j & 15] += s1(W[(j - 2) & 15]) +
                                             W[(j - 7) & 15] +
                                             s0(W[(j - 15) & 15]);
                        }
                }
                t1 = h + e1(e) + Ch(e, f, g) + sha512_K[i]   + W[(i & 15)];
                t2 = e0(a) + Maj(a, b, c);    d += t1;    h = t1 + t2;
                t1 = g + e1(d) + Ch(d, e, f) + sha512_K[i+1] + W[(i & 15) + 1];
                t2 = e0(h) + Maj(h, a, b);    c += t1;    g = t1 + t2;
                t1 = f + e1(c) + Ch(c, d, e) + sha512_K[i+2] + W[(i & 15) + 2];
                t2 = e0(g) + Maj(g, h, a);    b += t1;    f = t1 + t2;
                t1 = e + e1(b) + Ch(b, c, d) + sha512_K[i+3] + W[(i & 15) + 3];
                t2 = e0(f) + Maj(f, g, h);    a += t1;    e = t1 + t2;
                t1 = d + e1(a) + Ch(a, b, c) + sha512_K[i+4] + W[(i & 15) + 4];
                t2 = e0(e) + Maj(e, f, g);    h += t1;    d = t1 + t2;
                t1 = c + e1(h) + Ch(h, a, b) + sha512_K[i+5] + W[(i & 15) + 5];
                t2 = e0(d) + Maj(d, e, f);    g += t1;    c = t1 + t2;
                t1 = b + e1(g) + Ch(g, h, a) + sha512_K[i+6] + W[(i & 15) + 6];
                t2 = e0(c) + Maj(c, d, e);    f += t1;    b = t1 + t2;
                t1 = a + e1(f) + Ch(f, g, h) + sha512_K[i+7] + W[(i & 15) + 7];
                t2 = e0(b) + Maj(b, c, d);    e += t1;    a = t1 + t2;
        }

        state->h[0] += a;
        state->h[1] += b;
        state->h[2] += c;
        state->h[3] += d;
        state->h[4] += e;
        state->h[5] += f;
        state->h[6] += g;
        state->h[7] += h;
}

static void __maybe_unused
sha512_blocks_generic(struct sha512_block_state *state,
                      const u8 *data, size_t nblocks)
{
        do {
                sha512_block_generic(state, data);
                data += SHA512_BLOCK_SIZE;
        } while (--nblocks);
}

#ifdef CONFIG_CRYPTO_LIB_SHA512_ARCH
#include "sha512.h" /* $(SRCARCH)/sha512.h */
#else
#define sha512_blocks sha512_blocks_generic
#endif

static void __sha512_init(struct __sha512_ctx *ctx,
                          const struct sha512_block_state *iv,
                          u64 initial_bytecount)
{
        ctx->state = *iv;
        ctx->bytecount_lo = initial_bytecount;
        ctx->bytecount_hi = 0;
}

void sha384_init(struct sha384_ctx *ctx)
{
        __sha512_init(&ctx->ctx, &sha384_iv, 0);
}
EXPORT_SYMBOL_GPL(sha384_init);

void sha512_init(struct sha512_ctx *ctx)
{
        __sha512_init(&ctx->ctx, &sha512_iv, 0);
}
EXPORT_SYMBOL_GPL(sha512_init);

void __sha512_update(struct __sha512_ctx *ctx, const u8 *data, size_t len)
{
        size_t partial = ctx->bytecount_lo % SHA512_BLOCK_SIZE;

        if (check_add_overflow(ctx->bytecount_lo, len, &ctx->bytecount_lo))
                ctx->bytecount_hi++;

        if (partial + len >= SHA512_BLOCK_SIZE) {
                size_t nblocks;

                if (partial) {
                        size_t l = SHA512_BLOCK_SIZE - partial;

                        memcpy(&ctx->buf[partial], data, l);
                        data += l;
                        len -= l;

                        sha512_blocks(&ctx->state, ctx->buf, 1);
                }

                nblocks = len / SHA512_BLOCK_SIZE;
                len %= SHA512_BLOCK_SIZE;

                if (nblocks) {
                        sha512_blocks(&ctx->state, data, nblocks);
                        data += nblocks * SHA512_BLOCK_SIZE;
                }
                partial = 0;
        }
        if (len)
                memcpy(&ctx->buf[partial], data, len);
}
EXPORT_SYMBOL_GPL(__sha512_update);

static void __sha512_final(struct __sha512_ctx *ctx,
                           u8 *out, size_t digest_size)
{
        u64 bitcount_hi = (ctx->bytecount_hi << 3) | (ctx->bytecount_lo >> 61);
        u64 bitcount_lo = ctx->bytecount_lo << 3;
        size_t partial = ctx->bytecount_lo % SHA512_BLOCK_SIZE;

        ctx->buf[partial++] = 0x80;
        if (partial > SHA512_BLOCK_SIZE - 16) {
                memset(&ctx->buf[partial], 0, SHA512_BLOCK_SIZE - partial);
                sha512_blocks(&ctx->state, ctx->buf, 1);
                partial = 0;
        }
        memset(&ctx->buf[partial], 0, SHA512_BLOCK_SIZE - 16 - partial);
        *(__be64 *)&ctx->buf[SHA512_BLOCK_SIZE - 16] = cpu_to_be64(bitcount_hi);
        *(__be64 *)&ctx->buf[SHA512_BLOCK_SIZE - 8] = cpu_to_be64(bitcount_lo);
        sha512_blocks(&ctx->state, ctx->buf, 1);

        for (size_t i = 0; i < digest_size; i += 8)
                put_unaligned_be64(ctx->state.h[i / 8], out + i);
}

void sha384_final(struct sha384_ctx *ctx, u8 out[SHA384_DIGEST_SIZE])
{
        __sha512_final(&ctx->ctx, out, SHA384_DIGEST_SIZE);
        memzero_explicit(ctx, sizeof(*ctx));
}
EXPORT_SYMBOL_GPL(sha384_final);

void sha512_final(struct sha512_ctx *ctx, u8 out[SHA512_DIGEST_SIZE])
{
        __sha512_final(&ctx->ctx, out, SHA512_DIGEST_SIZE);
        memzero_explicit(ctx, sizeof(*ctx));
}
EXPORT_SYMBOL_GPL(sha512_final);

void sha384(const u8 *data, size_t len, u8 out[SHA384_DIGEST_SIZE])
{
        struct sha384_ctx ctx;

        sha384_init(&ctx);
        sha384_update(&ctx, data, len);
        sha384_final(&ctx, out);
}
EXPORT_SYMBOL_GPL(sha384);

void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE])
{
        struct sha512_ctx ctx;

        sha512_init(&ctx);
        sha512_update(&ctx, data, len);
        sha512_final(&ctx, out);
}
EXPORT_SYMBOL_GPL(sha512);

static void __hmac_sha512_preparekey(struct sha512_block_state *istate,
                                     struct sha512_block_state *ostate,
                                     const u8 *raw_key, size_t raw_key_len,
                                     const struct sha512_block_state *iv)
{
        union {
                u8 b[SHA512_BLOCK_SIZE];
                unsigned long w[SHA512_BLOCK_SIZE / sizeof(unsigned long)];
        } derived_key = { 0 };

        if (unlikely(raw_key_len > SHA512_BLOCK_SIZE)) {
                if (iv == &sha384_iv)
                        sha384(raw_key, raw_key_len, derived_key.b);
                else
                        sha512(raw_key, raw_key_len, derived_key.b);
        } else {
                memcpy(derived_key.b, raw_key, raw_key_len);
        }

        for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++)
                derived_key.w[i] ^= REPEAT_BYTE(HMAC_IPAD_VALUE);
        *istate = *iv;
        sha512_blocks(istate, derived_key.b, 1);

        for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++)
                derived_key.w[i] ^= REPEAT_BYTE(HMAC_OPAD_VALUE ^
                                                HMAC_IPAD_VALUE);
        *ostate = *iv;
        sha512_blocks(ostate, derived_key.b, 1);

        memzero_explicit(&derived_key, sizeof(derived_key));
}

void hmac_sha384_preparekey(struct hmac_sha384_key *key,
                            const u8 *raw_key, size_t raw_key_len)
{
        __hmac_sha512_preparekey(&key->key.istate, &key->key.ostate,
                                 raw_key, raw_key_len, &sha384_iv);
}
EXPORT_SYMBOL_GPL(hmac_sha384_preparekey);

void hmac_sha512_preparekey(struct hmac_sha512_key *key,
                            const u8 *raw_key, size_t raw_key_len)
{
        __hmac_sha512_preparekey(&key->key.istate, &key->key.ostate,
                                 raw_key, raw_key_len, &sha512_iv);
}
EXPORT_SYMBOL_GPL(hmac_sha512_preparekey);

void __hmac_sha512_init(struct __hmac_sha512_ctx *ctx,
                        const struct __hmac_sha512_key *key)
{
        __sha512_init(&ctx->sha_ctx, &key->istate, SHA512_BLOCK_SIZE);
        ctx->ostate = key->ostate;
}
EXPORT_SYMBOL_GPL(__hmac_sha512_init);

void hmac_sha384_init_usingrawkey(struct hmac_sha384_ctx *ctx,
                                  const u8 *raw_key, size_t raw_key_len)
{
        __hmac_sha512_preparekey(&ctx->ctx.sha_ctx.state, &ctx->ctx.ostate,
                                 raw_key, raw_key_len, &sha384_iv);
        ctx->ctx.sha_ctx.bytecount_lo = SHA512_BLOCK_SIZE;
        ctx->ctx.sha_ctx.bytecount_hi = 0;
}
EXPORT_SYMBOL_GPL(hmac_sha384_init_usingrawkey);

void hmac_sha512_init_usingrawkey(struct hmac_sha512_ctx *ctx,
                                  const u8 *raw_key, size_t raw_key_len)
{
        __hmac_sha512_preparekey(&ctx->ctx.sha_ctx.state, &ctx->ctx.ostate,
                                 raw_key, raw_key_len, &sha512_iv);
        ctx->ctx.sha_ctx.bytecount_lo = SHA512_BLOCK_SIZE;
        ctx->ctx.sha_ctx.bytecount_hi = 0;
}
EXPORT_SYMBOL_GPL(hmac_sha512_init_usingrawkey);

static void __hmac_sha512_final(struct __hmac_sha512_ctx *ctx,
                                u8 *out, size_t digest_size)
{
        /* Generate the padded input for the outer hash in ctx->sha_ctx.buf. */
        __sha512_final(&ctx->sha_ctx, ctx->sha_ctx.buf, digest_size);
        memset(&ctx->sha_ctx.buf[digest_size], 0,
               SHA512_BLOCK_SIZE - digest_size);
        ctx->sha_ctx.buf[digest_size] = 0x80;
        *(__be32 *)&ctx->sha_ctx.buf[SHA512_BLOCK_SIZE - 4] =
                cpu_to_be32(8 * (SHA512_BLOCK_SIZE + digest_size));

        /* Compute the outer hash, which gives the HMAC value. */
        sha512_blocks(&ctx->ostate, ctx->sha_ctx.buf, 1);
        for (size_t i = 0; i < digest_size; i += 8)
                put_unaligned_be64(ctx->ostate.h[i / 8], out + i);

        memzero_explicit(ctx, sizeof(*ctx));
}

void hmac_sha384_final(struct hmac_sha384_ctx *ctx,
                       u8 out[SHA384_DIGEST_SIZE])
{
        __hmac_sha512_final(&ctx->ctx, out, SHA384_DIGEST_SIZE);
}
EXPORT_SYMBOL_GPL(hmac_sha384_final);

void hmac_sha512_final(struct hmac_sha512_ctx *ctx,
                       u8 out[SHA512_DIGEST_SIZE])
{
        __hmac_sha512_final(&ctx->ctx, out, SHA512_DIGEST_SIZE);
}
EXPORT_SYMBOL_GPL(hmac_sha512_final);

void hmac_sha384(const struct hmac_sha384_key *key,
                 const u8 *data, size_t data_len, u8 out[SHA384_DIGEST_SIZE])
{
        struct hmac_sha384_ctx ctx;

        hmac_sha384_init(&ctx, key);
        hmac_sha384_update(&ctx, data, data_len);
        hmac_sha384_final(&ctx, out);
}
EXPORT_SYMBOL_GPL(hmac_sha384);

void hmac_sha512(const struct hmac_sha512_key *key,
                 const u8 *data, size_t data_len, u8 out[SHA512_DIGEST_SIZE])
{
        struct hmac_sha512_ctx ctx;

        hmac_sha512_init(&ctx, key);
        hmac_sha512_update(&ctx, data, data_len);
        hmac_sha512_final(&ctx, out);
}
EXPORT_SYMBOL_GPL(hmac_sha512);

void hmac_sha384_usingrawkey(const u8 *raw_key, size_t raw_key_len,
                             const u8 *data, size_t data_len,
                             u8 out[SHA384_DIGEST_SIZE])
{
        struct hmac_sha384_ctx ctx;

        hmac_sha384_init_usingrawkey(&ctx, raw_key, raw_key_len);
        hmac_sha384_update(&ctx, data, data_len);
        hmac_sha384_final(&ctx, out);
}
EXPORT_SYMBOL_GPL(hmac_sha384_usingrawkey);

void hmac_sha512_usingrawkey(const u8 *raw_key, size_t raw_key_len,
                             const u8 *data, size_t data_len,
                             u8 out[SHA512_DIGEST_SIZE])
{
        struct hmac_sha512_ctx ctx;

        hmac_sha512_init_usingrawkey(&ctx, raw_key, raw_key_len);
        hmac_sha512_update(&ctx, data, data_len);
        hmac_sha512_final(&ctx, out);
}
EXPORT_SYMBOL_GPL(hmac_sha512_usingrawkey);

#if defined(sha512_mod_init_arch) || defined(CONFIG_CRYPTO_FIPS)
static int __init sha512_mod_init(void)
{
#ifdef sha512_mod_init_arch
        sha512_mod_init_arch();
#endif
        if (fips_enabled) {
                /*
                 * FIPS cryptographic algorithm self-test.  As per the FIPS
                 * Implementation Guidance, testing HMAC-SHA512 satisfies the
                 * test requirement for SHA-384, SHA-512, and HMAC-SHA384 too.
                 */
                u8 mac[SHA512_DIGEST_SIZE];

                hmac_sha512_usingrawkey(fips_test_key, sizeof(fips_test_key),
                                        fips_test_data, sizeof(fips_test_data),
                                        mac);
                if (memcmp(fips_test_hmac_sha512_value, mac, sizeof(mac)) != 0)
                        panic("sha512: FIPS self-test failed\n");
        }
        return 0;
}
subsys_initcall(sha512_mod_init);

static void __exit sha512_mod_exit(void)
{
}
module_exit(sha512_mod_exit);
#endif

MODULE_DESCRIPTION("SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library functions");
MODULE_LICENSE("GPL");




















































































































































































































































   25 




   25 

















    6 




    6 



















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
/* SPDX-License-Identifier: GPL-2.0 */
/*
 *        Declarations of X.25 Packet Layer type objects.
 *
 *         History
 *        nov/17/96        Jonathan Naylor          Initial version.                
 *        mar/20/00        Daniela Squassoni Disabling/enabling of facilities 
 *                                          negotiation.
 */

#ifndef _X25_H
#define _X25_H 
#include <linux/x25.h>
#include <linux/slab.h>
#include <linux/refcount.h>
#include <net/sock.h>

#define        X25_ADDR_LEN                        16

#define        X25_MAX_L2_LEN                        18        /* 802.2 LLC */

#define        X25_STD_MIN_LEN                        3
#define        X25_EXT_MIN_LEN                        4

#define        X25_GFI_SEQ_MASK                0x30
#define        X25_GFI_STDSEQ                        0x10
#define        X25_GFI_EXTSEQ                        0x20

#define        X25_Q_BIT                        0x80
#define        X25_D_BIT                        0x40
#define        X25_STD_M_BIT                        0x10
#define        X25_EXT_M_BIT                        0x01

#define        X25_CALL_REQUEST                0x0B
#define        X25_CALL_ACCEPTED                0x0F
#define        X25_CLEAR_REQUEST                0x13
#define        X25_CLEAR_CONFIRMATION                0x17
#define        X25_DATA                        0x00
#define        X25_INTERRUPT                        0x23
#define        X25_INTERRUPT_CONFIRMATION        0x27
#define        X25_RR                                0x01
#define        X25_RNR                                0x05
#define        X25_REJ                                0x09
#define        X25_RESET_REQUEST                0x1B
#define        X25_RESET_CONFIRMATION                0x1F
#define        X25_REGISTRATION_REQUEST        0xF3
#define        X25_REGISTRATION_CONFIRMATION        0xF7
#define        X25_RESTART_REQUEST                0xFB
#define        X25_RESTART_CONFIRMATION        0xFF
#define        X25_DIAGNOSTIC                        0xF1
#define        X25_ILLEGAL                        0xFD

/* Define the various conditions that may exist */

#define        X25_COND_ACK_PENDING        0x01
#define        X25_COND_OWN_RX_BUSY        0x02
#define        X25_COND_PEER_RX_BUSY        0x04

/* Define Link State constants. */
enum {
        X25_STATE_0,                /* Ready */
        X25_STATE_1,                /* Awaiting Call Accepted */
        X25_STATE_2,                /* Awaiting Clear Confirmation */
        X25_STATE_3,                /* Data Transfer */
        X25_STATE_4,                /* Awaiting Reset Confirmation */
        X25_STATE_5                /* Call Accepted / Call Connected pending */
};

enum {
        X25_LINK_STATE_0,
        X25_LINK_STATE_1,
        X25_LINK_STATE_2,
        X25_LINK_STATE_3
};

#define X25_DEFAULT_T20                (180 * HZ)                /* Default T20 value */
#define X25_DEFAULT_T21                (200 * HZ)                /* Default T21 value */
#define X25_DEFAULT_T22                (180 * HZ)                /* Default T22 value */
#define        X25_DEFAULT_T23                (180 * HZ)                /* Default T23 value */
#define        X25_DEFAULT_T2                (3   * HZ)                /* Default ack holdback value */

#define        X25_DEFAULT_WINDOW_SIZE        2                        /* Default Window Size        */
#define        X25_DEFAULT_PACKET_SIZE        X25_PS128                /* Default Packet Size */
#define        X25_DEFAULT_THROUGHPUT        0x0A                        /* Default Throughput */
#define        X25_DEFAULT_REVERSE        0x00                        /* Default Reverse Charging */

#define X25_SMODULUS                 8
#define        X25_EMODULUS                128

/*
 *        X.25 Facilities constants.
 */

#define        X25_FAC_CLASS_MASK        0xC0

#define        X25_FAC_CLASS_A                0x00
#define        X25_FAC_CLASS_B                0x40
#define        X25_FAC_CLASS_C                0x80
#define        X25_FAC_CLASS_D                0xC0

#define        X25_FAC_REVERSE                0x01                        /* also fast select */
#define        X25_FAC_THROUGHPUT        0x02
#define        X25_FAC_PACKET_SIZE        0x42
#define        X25_FAC_WINDOW_SIZE        0x43

#define X25_MAX_FAC_LEN         60
#define        X25_MAX_CUD_LEN                128

#define X25_FAC_CALLING_AE         0xCB
#define X25_FAC_CALLED_AE         0xC9

#define X25_MARKER                 0x00
#define X25_DTE_SERVICES         0x0F
#define X25_MAX_AE_LEN                 40                        /* Max num of semi-octets in AE - OSI Nw */
#define X25_MAX_DTE_FACIL_LEN        21                        /* Max length of DTE facility params */

/* Bitset in x25_sock->flags for misc flags */
#define X25_Q_BIT_FLAG                0
#define X25_INTERRUPT_FLAG        1
#define X25_ACCPT_APPRV_FLAG        2

/**
 *        struct x25_route - x25 routing entry
 *        @node - entry in x25_list_lock
 *        @address - Start of address range
 *        @sigdigits - Number of sig digits
 *        @dev - More than one for MLP
 *        @refcnt - reference counter
 */
struct x25_route {
        struct list_head        node;                
        struct x25_address        address;
        unsigned int                sigdigits;
        struct net_device        *dev;
        refcount_t                refcnt;
};

struct x25_neigh {
        struct list_head        node;
        struct net_device        *dev;
        unsigned int                state;
        unsigned int                extended;
        struct sk_buff_head        queue;
        unsigned long                t20;
        struct timer_list        t20timer;
        unsigned long                global_facil_mask;
        refcount_t                refcnt;
};

struct x25_sock {
        struct sock                sk;
        struct x25_address        source_addr, dest_addr;
        struct x25_neigh        *neighbour;
        unsigned int                lci, cudmatchlength;
        unsigned char                state, condition;
        unsigned short                vs, vr, va, vl;
        unsigned long                t2, t21, t22, t23;
        unsigned short                fraglen;
        unsigned long                flags;
        struct sk_buff_head        ack_queue;
        struct sk_buff_head        fragment_queue;
        struct sk_buff_head        interrupt_in_queue;
        struct sk_buff_head        interrupt_out_queue;
        struct timer_list        timer;
        struct x25_causediag        causediag;
        struct x25_facilities        facilities;
        struct x25_dte_facilities dte_facilities;
        struct x25_calluserdata        calluserdata;
        unsigned long                 vc_facil_mask;        /* inc_call facilities mask */
};

struct x25_forward {
        struct list_head        node;
        unsigned int                lci;
        struct net_device        *dev1;
        struct net_device        *dev2;
        atomic_t                refcnt;
};

#define x25_sk(ptr) container_of_const(ptr, struct x25_sock, sk)

/* af_x25.c */
extern int  sysctl_x25_restart_request_timeout;
extern int  sysctl_x25_call_request_timeout;
extern int  sysctl_x25_reset_request_timeout;
extern int  sysctl_x25_clear_request_timeout;
extern int  sysctl_x25_ack_holdback_timeout;
extern int  sysctl_x25_forward;

int x25_parse_address_block(struct sk_buff *skb,
                            struct x25_address *called_addr,
                            struct x25_address *calling_addr);

int x25_addr_ntoa(unsigned char *, struct x25_address *, struct x25_address *);
int x25_addr_aton(unsigned char *, struct x25_address *, struct x25_address *);
struct sock *x25_find_socket(unsigned int, struct x25_neigh *);
void x25_destroy_socket_from_timer(struct sock *);
int x25_rx_call_request(struct sk_buff *, struct x25_neigh *, unsigned int);
void x25_kill_by_neigh(struct x25_neigh *);

/* x25_dev.c */
void x25_send_frame(struct sk_buff *, struct x25_neigh *);
int x25_lapb_receive_frame(struct sk_buff *, struct net_device *,
                           struct packet_type *, struct net_device *);
void x25_establish_link(struct x25_neigh *);

/* x25_facilities.c */
int x25_parse_facilities(struct sk_buff *, struct x25_facilities *,
                         struct x25_dte_facilities *, unsigned long *);
int x25_create_facilities(unsigned char *, struct x25_facilities *,
                          struct x25_dte_facilities *, unsigned long);
int x25_negotiate_facilities(struct sk_buff *, struct sock *,
                             struct x25_facilities *,
                             struct x25_dte_facilities *);
void x25_limit_facilities(struct x25_facilities *, struct x25_neigh *);

/* x25_forward.c */
void x25_clear_forward_by_lci(unsigned int lci);
void x25_clear_forward_by_dev(struct net_device *);
int x25_forward_data(int, struct x25_neigh *, struct sk_buff *);
int x25_forward_call(struct x25_address *, struct x25_neigh *, struct sk_buff *,
                     int);

/* x25_in.c */
int x25_process_rx_frame(struct sock *, struct sk_buff *);
int x25_backlog_rcv(struct sock *, struct sk_buff *);

/* x25_link.c */
void x25_link_control(struct sk_buff *, struct x25_neigh *, unsigned short);
void x25_link_device_up(struct net_device *);
void x25_link_device_down(struct net_device *);
void x25_link_established(struct x25_neigh *);
void x25_link_terminated(struct x25_neigh *);
void x25_transmit_clear_request(struct x25_neigh *, unsigned int,
                                unsigned char);
void x25_transmit_link(struct sk_buff *, struct x25_neigh *);
int x25_subscr_ioctl(unsigned int, void __user *);
struct x25_neigh *x25_get_neigh(struct net_device *);
void x25_link_free(void);

/* x25_neigh.c */
static __inline__ void x25_neigh_hold(struct x25_neigh *nb)
{
        refcount_inc(&nb->refcnt);
}

static __inline__ void x25_neigh_put(struct x25_neigh *nb)
{
        if (refcount_dec_and_test(&nb->refcnt))
                kfree(nb);
}

/* x25_out.c */
int x25_output(struct sock *, struct sk_buff *);
void x25_kick(struct sock *);
void x25_enquiry_response(struct sock *);

/* x25_route.c */
struct x25_route *x25_get_route(struct x25_address *addr);
struct net_device *x25_dev_get(char *);
void x25_route_device_down(struct net_device *dev);
int x25_route_ioctl(unsigned int, void __user *);
void x25_route_free(void);

static __inline__ void x25_route_hold(struct x25_route *rt)
{
        refcount_inc(&rt->refcnt);
}

static __inline__ void x25_route_put(struct x25_route *rt)
{
        if (refcount_dec_and_test(&rt->refcnt))
                kfree(rt);
}

/* x25_subr.c */
void x25_clear_queues(struct sock *);
void x25_frames_acked(struct sock *, unsigned short);
void x25_requeue_frames(struct sock *);
int x25_validate_nr(struct sock *, unsigned short);
void x25_write_internal(struct sock *, int);
int x25_decode(struct sock *, struct sk_buff *, int *, int *, int *, int *,
               int *);
void x25_disconnect(struct sock *, int, unsigned char, unsigned char);

/* x25_timer.c */
void x25_init_timers(struct sock *sk);
void x25_start_heartbeat(struct sock *);
void x25_start_t2timer(struct sock *);
void x25_start_t21timer(struct sock *);
void x25_start_t22timer(struct sock *);
void x25_start_t23timer(struct sock *);
void x25_stop_heartbeat(struct sock *);
void x25_stop_timer(struct sock *);
unsigned long x25_display_timer(struct sock *);
void x25_check_rbuf(struct sock *);

/* sysctl_net_x25.c */
#ifdef CONFIG_SYSCTL
int x25_register_sysctl(void);
void x25_unregister_sysctl(void);
#else
static inline int x25_register_sysctl(void) { return 0; };
static inline void x25_unregister_sysctl(void) {};
#endif /* CONFIG_SYSCTL */

struct x25_skb_cb {
        unsigned int flags;
};
#define X25_SKB_CB(s) ((struct x25_skb_cb *) ((s)->cb))

extern struct hlist_head x25_list;
extern rwlock_t x25_list_lock;
extern struct list_head x25_route_list;
extern rwlock_t x25_route_list_lock;
extern struct list_head x25_forward_list;
extern rwlock_t x25_forward_list_lock;
extern struct list_head x25_neigh_list;
extern rwlock_t x25_neigh_list_lock;

int x25_proc_init(void);
void x25_proc_exit(void);
#endif


























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   22 
   22 























































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* internal AFS stuff
 *
 * Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/ktime.h>
#include <linux/fs.h>
#include <linux/filelock.h>
#include <linux/pagemap.h>
#include <linux/rxrpc.h>
#include <linux/key.h>
#include <linux/workqueue.h>
#include <linux/sched.h>
#include <linux/fscache.h>
#include <linux/backing-dev.h>
#include <linux/uuid.h>
#include <linux/mm_types.h>
#include <linux/dns_resolver.h>
#include <crypto/krb5.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/sock.h>
#include <net/af_rxrpc.h>

#include "afs.h"
#include "afs_vl.h"

#define AFS_CELL_MAX_ADDRS 15

struct afs_call;
struct afs_vnode;
struct afs_server_probe;

/*
 * Partial file-locking emulation mode.  (The problem being that AFS3 only
 * allows whole-file locks and no upgrading/downgrading).
 */
enum afs_flock_mode {
        afs_flock_mode_unset,
        afs_flock_mode_local,        /* Local locking only */
        afs_flock_mode_openafs,        /* Don't get server lock for a partial lock */
        afs_flock_mode_strict,        /* Always get a server lock for a partial lock */
        afs_flock_mode_write,        /* Get an exclusive server lock for a partial lock */
};

struct afs_fs_context {
        bool                        force;                /* T to force cell type */
        bool                        autocell;        /* T if set auto mount operation */
        bool                        dyn_root;        /* T if dynamic root */
        bool                        no_cell;        /* T if the source is "none" (for dynroot) */
        enum afs_flock_mode        flock_mode;        /* Partial file-locking emulation mode */
        afs_voltype_t                type;                /* type of volume requested */
        unsigned int                volnamesz;        /* size of volume name */
        const char                *volname;        /* name of volume to mount */
        struct afs_net                *net;                /* the AFS net namespace stuff */
        struct afs_cell                *cell;                /* cell in which to find volume */
        struct afs_volume        *volume;        /* volume record */
        struct key                *key;                /* key to use for secure mounting */
};

enum afs_call_state {
        AFS_CALL_CL_REQUESTING,                /* Client: Request is being sent */
        AFS_CALL_CL_AWAIT_REPLY,        /* Client: Awaiting reply */
        AFS_CALL_CL_PROC_REPLY,                /* Client: rxrpc call complete; processing reply */
        AFS_CALL_SV_AWAIT_OP_ID,        /* Server: Awaiting op ID */
        AFS_CALL_SV_AWAIT_REQUEST,        /* Server: Awaiting request data */
        AFS_CALL_SV_REPLYING,                /* Server: Replying */
        AFS_CALL_SV_AWAIT_ACK,                /* Server: Awaiting final ACK */
        AFS_CALL_COMPLETE,                /* Completed or failed */
};

/*
 * Address preferences.
 */
struct afs_addr_preference {
        union {
                struct in_addr        ipv4_addr;        /* AF_INET address to compare against */
                struct in6_addr        ipv6_addr;        /* AF_INET6 address to compare against */
        };
        sa_family_t                family;                /* Which address to use */
        u16                        prio;                /* Priority */
        u8                        subnet_mask;        /* How many bits to compare */
};

struct afs_addr_preference_list {
        struct rcu_head                rcu;
        u16                        version;        /* Incremented when prefs list changes */
        u8                        ipv6_off;        /* Offset of IPv6 addresses */
        u8                        nr;                /* Number of addresses in total */
        u8                        max_prefs;        /* Number of prefs allocated */
        struct afs_addr_preference prefs[] __counted_by(max_prefs);
};

struct afs_address {
        struct rxrpc_peer        *peer;
        short                        last_error;        /* Last error from this address */
        u16                        prio;                /* Address priority */
};

/*
 * List of server addresses.
 */
struct afs_addr_list {
        struct rcu_head                rcu;
        refcount_t                usage;
        u32                        version;        /* Version */
        unsigned int                debug_id;
        unsigned int                addr_pref_version; /* Version of address preference list */
        unsigned char                max_addrs;
        unsigned char                nr_addrs;
        unsigned char                preferred;        /* Preferred address */
        unsigned char                nr_ipv4;        /* Number of IPv4 addresses */
        enum dns_record_source        source:8;
        enum dns_lookup_status        status:8;
        unsigned long                probe_failed;        /* Mask of addrs that failed locally/ICMP */
        unsigned long                responded;        /* Mask of addrs that responded */
        struct afs_address        addrs[] __counted_by(max_addrs);
#define AFS_MAX_ADDRESSES ((unsigned int)(sizeof(unsigned long) * 8))
};

/*
 * a record of an in-progress RxRPC call
 */
struct afs_call {
        const struct afs_call_type *type;        /* type of call */
        wait_queue_head_t        waitq;                /* processes awaiting completion */
        struct work_struct        async_work;        /* async I/O processor */
        struct work_struct        work;                /* actual work processor */
        struct work_struct        free_work;        /* Deferred free processor */
        struct rxrpc_call        *rxcall;        /* RxRPC call handle */
        struct rxrpc_peer        *peer;                /* Remote endpoint */
        struct key                *key;                /* security for this call */
        struct afs_net                *net;                /* The network namespace */
        struct afs_server        *server;        /* The fileserver record if fs op (pins ref) */
        struct afs_vlserver        *vlserver;        /* The vlserver record if vl op */
        void                        *request;        /* request data (first part) */
        size_t                        iov_len;        /* Size of *iter to be used */
        struct iov_iter                def_iter;        /* Default buffer/data iterator */
        struct iov_iter                *write_iter;        /* Iterator defining write to be made */
        struct iov_iter                *iter;                /* Iterator currently in use */
        union {        /* Convenience for ->def_iter */
                struct kvec        kvec[1];
                struct bio_vec        bvec[1];
        };
        void                        *buffer;        /* reply receive buffer */
        union {
                struct afs_endpoint_state *probe;
                struct afs_addr_list        *vl_probe;
                struct afs_addr_list        *ret_alist;
                struct afs_vldb_entry        *ret_vldb;
                char                        *ret_str;
        };
        struct afs_fid                fid;                /* Primary vnode ID (or all zeroes) */
        unsigned char                probe_index;        /* Address in ->probe_alist */
        struct afs_operation        *op;
        unsigned int                server_index;
        refcount_t                ref;
        enum afs_call_state        state;
        spinlock_t                state_lock;
        int                        error;                /* error code */
        u32                        abort_code;        /* Remote abort ID or 0 */
        unsigned long long        remaining;        /* How much is left to receive */
        unsigned int                max_lifespan;        /* Maximum lifespan in secs to set if not 0 */
        unsigned                request_size;        /* size of request data */
        unsigned                reply_max;        /* maximum size of reply */
        unsigned                count2;                /* count used in unmarshalling */
        unsigned char                unmarshall;        /* unmarshalling phase */
        bool                        drop_ref;        /* T if need to drop ref for incoming call */
        bool                        need_attention;        /* T if RxRPC poked us */
        bool                        async;                /* T if asynchronous */
        bool                        upgrade;        /* T to request service upgrade */
        bool                        intr;                /* T if interruptible */
        bool                        unmarshalling_error; /* T if an unmarshalling error occurred */
        bool                        responded;        /* Got a response from the call (may be abort) */
        u8                        security_ix;        /* Security class */
        u16                        service_id;        /* Actual service ID (after upgrade) */
        unsigned int                debug_id;        /* Trace ID */
        u32                        enctype;        /* Security encoding type */
        u32                        operation_ID;        /* operation ID for an incoming call */
        u32                        count;                /* count for use in unmarshalling */
        union {                                        /* place to extract temporary data */
                struct {
                        __be32        tmp_u;
                        __be32        tmp;
                } __attribute__((packed));
                __be64                tmp64;
        };
        ktime_t                        issue_time;        /* Time of issue of operation */
};

struct afs_call_type {
        const char *name;
        unsigned int op; /* Really enum afs_fs_operation */

        /* deliver request or reply data to an call
         * - returning an error will cause the call to be aborted
         */
        int (*deliver)(struct afs_call *call);

        /* clean up a call */
        void (*destructor)(struct afs_call *call);

        /* Async receive processing function */
        void (*async_rx)(struct work_struct *work);

        /* Work function */
        void (*work)(struct work_struct *work);

        /* Call done function (gets called immediately on success or failure) */
        void (*done)(struct afs_call *call);

        /* Handle a call being immediately cancelled. */
        void (*immediate_cancel)(struct afs_call *call);
};

/*
 * Key available for writeback on a file.
 */
struct afs_wb_key {
        refcount_t                usage;
        struct key                *key;
        struct list_head        vnode_link;        /* Link in vnode->wb_keys */
};

/*
 * AFS open file information record.  Pointed to by file->private_data.
 */
struct afs_file {
        struct key                *key;                /* The key this file was opened with */
        struct afs_wb_key        *wb;                /* Writeback key record for this file */
};

static inline struct key *afs_file_key(struct file *file)
{
        struct afs_file *af = file->private_data;

        return af->key;
}

/*
 * AFS superblock private data
 * - there's one superblock per volume
 */
struct afs_super_info {
        struct net                *net_ns;        /* Network namespace */
        struct afs_cell                *cell;                /* The cell in which the volume resides */
        struct afs_volume        *volume;        /* volume record */
        enum afs_flock_mode        flock_mode:8;        /* File locking emulation mode */
        bool                        dyn_root;        /* True if dynamic root */
};

static inline struct afs_super_info *AFS_FS_S(struct super_block *sb)
{
        return sb->s_fs_info;
}

extern struct file_system_type afs_fs_type;

/*
 * Set of substitutes for @sys.
 */
struct afs_sysnames {
#define AFS_NR_SYSNAME 16
        char                        *subs[AFS_NR_SYSNAME];
        refcount_t                usage;
        unsigned short                nr;
        char                        blank[1];
};

/*
 * AFS network namespace record.
 */
struct afs_net {
        struct net                *net;                /* Backpointer to the owning net namespace */
        struct afs_uuid                uuid;
        bool                        live;                /* F if this namespace is being removed */

        /* AF_RXRPC I/O stuff */
        struct socket                *socket;
        struct afs_call                *spare_incoming_call;
        struct work_struct        charge_preallocation_work;
        struct work_struct        rx_oob_work;
        struct mutex                socket_mutex;
        atomic_t                nr_outstanding_calls;
        atomic_t                nr_superblocks;

        /* Cell database */
        struct rb_root                cells;
        struct idr                cells_dyn_ino;        /* cell->dynroot_ino mapping */
        struct afs_cell __rcu        *ws_cell;
        atomic_t                cells_outstanding;
        struct rw_semaphore        cells_lock;
        struct mutex                cells_alias_lock;

        struct mutex                proc_cells_lock;
        struct hlist_head        proc_cells;

        /* Known servers.  Theoretically each fileserver can only be in one
         * cell, but in practice, people create aliases and subsets and there's
         * no easy way to distinguish them.
         */
        seqlock_t                fs_lock;        /* For fs_probe_*, fs_proc */
        struct list_head        fs_probe_fast;        /* List of afs_server to probe at 30s intervals */
        struct list_head        fs_probe_slow;        /* List of afs_server to probe at 5m intervals */
        struct hlist_head        fs_proc;        /* procfs servers list */

        struct key                *fs_cm_token_key; /* Key for creating CM tokens */
        struct work_struct        fs_prober;
        struct timer_list        fs_probe_timer;
        atomic_t                servers_outstanding;

        /* File locking renewal management */
        struct mutex                lock_manager_mutex;

        /* Misc */
        struct super_block        *dynroot_sb;        /* Dynamic root mount superblock */
        struct proc_dir_entry        *proc_afs;        /* /proc/net/afs directory */
        struct afs_sysnames        *sysnames;
        rwlock_t                sysnames_lock;
        struct afs_addr_preference_list __rcu *address_prefs;
        u16                        address_pref_version;

        /* Statistics counters */
        atomic_t                n_lookup;        /* Number of lookups done */
        atomic_t                n_reval;        /* Number of dentries needing revalidation */
        atomic_t                n_inval;        /* Number of invalidations by the server */
        atomic_t                n_relpg;        /* Number of invalidations by release_folio */
        atomic_t                n_read_dir;        /* Number of directory pages read */
        atomic_t                n_dir_cr;        /* Number of directory entry creation edits */
        atomic_t                n_dir_rm;        /* Number of directory entry removal edits */
        atomic_t                n_stores;        /* Number of store ops */
        atomic_long_t                n_store_bytes;        /* Number of bytes stored */
        atomic_long_t                n_fetch_bytes;        /* Number of bytes fetched */
        atomic_t                n_fetches;        /* Number of data fetch ops */
};

extern const char afs_init_sysname[];

enum afs_cell_state {
        AFS_CELL_SETTING_UP,
        AFS_CELL_UNLOOKED,
        AFS_CELL_ACTIVE,
        AFS_CELL_REMOVING,
        AFS_CELL_DEAD,
};

/*
 * AFS cell record.
 *
 * This is a tricky concept to get right as it is possible to create aliases
 * simply by pointing AFSDB/SRV records for two names at the same set of VL
 * servers; it is also possible to do things like setting up two sets of VL
 * servers, one of which provides a superset of the volumes provided by the
 * other (for internal/external division, for example).
 *
 * Cells only exist in the sense that (a) a cell's name maps to a set of VL
 * servers and (b) a cell's name is used by the client to select the key to use
 * for authentication and encryption.  The cell name is not typically used in
 * the protocol.
 *
 * Two cells are determined to be aliases if they have an explicit alias (YFS
 * only), share any VL servers in common or have at least one volume in common.
 * "In common" means that the address list of the VL servers or the fileservers
 * share at least one endpoint.
 */
struct afs_cell {
        union {
                struct rcu_head        rcu;
                struct rb_node        net_node;        /* Node in net->cells */
        };
        struct afs_net                *net;
        struct afs_cell                *alias_of;        /* The cell this is an alias of */
        struct afs_volume        *root_volume;        /* The root.cell volume if there is one */
        struct key                *anonymous_key;        /* anonymous user key for this cell */
        struct work_struct        destroyer;        /* Destroyer for cell */
        struct work_struct        manager;        /* Manager for init/deinit/dns */
        struct timer_list        management_timer; /* General management timer */
        struct hlist_node        proc_link;        /* /proc cell list link */
        time64_t                dns_expiry;        /* Time AFSDB/SRV record expires */
        time64_t                last_inactive;        /* Time of last drop of usage count */
        refcount_t                ref;                /* Struct refcount */
        atomic_t                active;                /* Active usage counter */
        unsigned long                flags;
#define AFS_CELL_FL_NO_GC        0                /* The cell was added manually, don't auto-gc */
#define AFS_CELL_FL_DO_LOOKUP        1                /* DNS lookup requested */
#define AFS_CELL_FL_CHECK_ALIAS        2                /* Need to check for aliases */
#define AFS_CELL_FL_HAVE_INO        3                /* Have dynroot_ino */
        enum afs_cell_state        state;
        short                        error;
        enum dns_record_source        dns_source:8;        /* Latest source of data from lookup */
        enum dns_lookup_status        dns_status:8;        /* Latest status of data from lookup */
        unsigned int                dns_lookup_count; /* Counter of DNS lookups */
        unsigned int                debug_id;
        unsigned int                dynroot_ino;        /* Inode numbers for dynroot (a pair) */

        /* The volumes belonging to this cell */
        struct rw_semaphore        vs_lock;        /* Lock for server->volumes */
        struct rb_root                volumes;        /* Tree of volumes on this server */
        struct hlist_head        proc_volumes;        /* procfs volume list */
        seqlock_t                volume_lock;        /* For volumes */

        /* Active fileserver interaction state. */
        struct rb_root                fs_servers;        /* afs_server (by server UUID) */
        struct rw_semaphore        fs_lock;        /* For fs_servers  */

        /* VL server list. */
        rwlock_t                vl_servers_lock; /* Lock on vl_servers */
        struct afs_vlserver_list __rcu *vl_servers;

        u8                        name_len;        /* Length of name */
        char                        *name;                /* Cell name, case-flattened and NUL-padded */
        char                        *key_desc;        /* Authentication key description */
};

/*
 * Volume Location server record.
 */
struct afs_vlserver {
        struct rcu_head                rcu;
        struct afs_addr_list        __rcu *addresses; /* List of addresses for this VL server */
        unsigned long                flags;
#define AFS_VLSERVER_FL_PROBED        0                /* The VL server has been probed */
#define AFS_VLSERVER_FL_PROBING        1                /* VL server is being probed */
#define AFS_VLSERVER_FL_IS_YFS        2                /* Server is YFS not AFS */
#define AFS_VLSERVER_FL_RESPONDING 3                /* VL server is responding */
        rwlock_t                lock;                /* Lock on addresses */
        refcount_t                ref;
        unsigned int                rtt;                /* Server's current RTT in uS */
        unsigned int                debug_id;

        /* Probe state */
        wait_queue_head_t        probe_wq;
        atomic_t                probe_outstanding;
        spinlock_t                probe_lock;
        struct {
                unsigned int        rtt;                /* Best RTT in uS (or UINT_MAX) */
                u32                abort_code;
                short                error;
                unsigned short        flags;
#define AFS_VLSERVER_PROBE_RESPONDED                0x01 /* At least once response (may be abort) */
#define AFS_VLSERVER_PROBE_IS_YFS                0x02 /* The peer appears to be YFS */
#define AFS_VLSERVER_PROBE_NOT_YFS                0x04 /* The peer appears not to be YFS */
#define AFS_VLSERVER_PROBE_LOCAL_FAILURE        0x08 /* A local failure prevented a probe */
        } probe;

        u16                        service_id;        /* Service ID we're using */
        u16                        port;
        u16                        name_len;        /* Length of name */
        char                        name[];                /* Server name, case-flattened */
};

/*
 * Weighted list of Volume Location servers.
 */
struct afs_vlserver_entry {
        u16                        priority;        /* Preference (as SRV) */
        u16                        weight;                /* Weight (as SRV) */
        enum dns_record_source        source:8;
        enum dns_lookup_status        status:8;
        struct afs_vlserver        *server;
};

struct afs_vlserver_list {
        struct rcu_head                rcu;
        refcount_t                ref;
        u8                        nr_servers;
        u8                        index;                /* Server currently in use */
        u8                        preferred;        /* Preferred server */
        enum dns_record_source        source:8;
        enum dns_lookup_status        status:8;
        rwlock_t                lock;
        struct afs_vlserver_entry servers[];
};

/*
 * Cached VLDB entry.
 *
 * This is pointed to by cell->vldb_entries, indexed by name.
 */
struct afs_vldb_entry {
        afs_volid_t                vid[3];                /* Volume IDs for R/W, R/O and Bak volumes */

        unsigned long                flags;
#define AFS_VLDB_HAS_RW                0                /* - R/W volume exists */
#define AFS_VLDB_HAS_RO                1                /* - R/O volume exists */
#define AFS_VLDB_HAS_BAK        2                /* - Backup volume exists */
#define AFS_VLDB_QUERY_VALID        3                /* - Record is valid */
#define AFS_VLDB_QUERY_ERROR        4                /* - VL server returned error */

        uuid_t                        fs_server[AFS_NMAXNSERVERS];
        u32                        addr_version[AFS_NMAXNSERVERS]; /* Registration change counters */
        u8                        fs_mask[AFS_NMAXNSERVERS];
#define AFS_VOL_VTM_RW        0x01 /* R/W version of the volume is available (on this server) */
#define AFS_VOL_VTM_RO        0x02 /* R/O version of the volume is available (on this server) */
#define AFS_VOL_VTM_BAK        0x04 /* backup version of the volume is available (on this server) */
        u8                        vlsf_flags[AFS_NMAXNSERVERS];
        short                        error;
        u8                        nr_servers;        /* Number of server records */
        u8                        name_len;
        u8                        name[AFS_MAXVOLNAME + 1]; /* NUL-padded volume name */
};

/*
 * Fileserver endpoint state.  The records the addresses of a fileserver's
 * endpoints and the state and result of a round of probing on them.  This
 * allows the rotation algorithm to access those results without them being
 * erased by a subsequent round of probing.
 */
struct afs_endpoint_state {
        struct rcu_head                rcu;
        struct afs_addr_list        *addresses;        /* The addresses being probed */
        unsigned long                responsive_set;        /* Bitset of responsive endpoints */
        unsigned long                failed_set;        /* Bitset of endpoints we failed to probe */
        refcount_t                ref;
        unsigned int                server_id;        /* Debug ID of server */
        unsigned int                probe_seq;        /* Probe sequence (from server::probe_counter) */
        atomic_t                nr_probing;        /* Number of outstanding probes */
        unsigned int                rtt;                /* Best RTT in uS (or UINT_MAX) */
        s32                        abort_code;
        short                        error;
        unsigned long                flags;
#define AFS_ESTATE_RESPONDED        0                /* Set if the server responded */
#define AFS_ESTATE_SUPERSEDED        1                /* Set if this record has been superseded */
#define AFS_ESTATE_IS_YFS        2                /* Set if probe upgraded to YFS */
#define AFS_ESTATE_NOT_YFS        3                /* Set if probe didn't upgrade to YFS */
#define AFS_ESTATE_LOCAL_FAILURE 4                /* Set if there was a local failure (eg. ENOMEM) */
};

/*
 * Record of fileserver with which we're actively communicating.
 */
struct afs_server {
        struct rcu_head                rcu;
        union {
                uuid_t                uuid;                /* Server ID */
                struct afs_uuid        _uuid;
        };

        struct afs_cell                *cell;                /* Cell to which belongs (pins ref) */
        struct rb_node                uuid_rb;        /* Link in cell->fs_servers */
        struct list_head        probe_link;        /* Link in net->fs_probe_* */
        struct hlist_node        proc_link;        /* Link in net->fs_proc */
        struct list_head        volumes;        /* RCU list of afs_server_entry objects */
        struct work_struct        destroyer;        /* Work item to try and destroy a server */
        struct timer_list        timer;                /* Management timer */
        struct mutex                cm_token_lock;        /* Lock governing creation of appdata */
        struct krb5_buffer        cm_rxgk_appdata; /* Appdata to be included in RESPONSE packet */
        time64_t                unuse_time;        /* Time at which last unused */
        unsigned long                flags;
#define AFS_SERVER_FL_RESPONDING 0                /* The server is responding */
#define AFS_SERVER_FL_UPDATING        1
#define AFS_SERVER_FL_NEEDS_UPDATE 2                /* Fileserver address list is out of date */
#define AFS_SERVER_FL_UNCREATED        3                /* The record needs creating */
#define AFS_SERVER_FL_CREATING        4                /* The record is being created */
#define AFS_SERVER_FL_EXPIRED        5                /* The record has expired */
#define AFS_SERVER_FL_NOT_FOUND        6                /* VL server says no such server */
#define AFS_SERVER_FL_VL_FAIL        7                /* Failed to access VL server */
#define AFS_SERVER_FL_MAY_HAVE_CB 8                /* May have callbacks on this fileserver */
#define AFS_SERVER_FL_IS_YFS        16                /* Server is YFS not AFS */
#define AFS_SERVER_FL_NO_IBULK        17                /* Fileserver doesn't support FS.InlineBulkStatus */
#define AFS_SERVER_FL_NO_RM2        18                /* Fileserver doesn't support YFS.RemoveFile2 */
#define AFS_SERVER_FL_HAS_FS64        19                /* Fileserver supports FS.{Fetch,Store}Data64 */
#define AFS_SERVER_FL_NO_RENAME2 20                /* YFS Fileserver doesn't support enhanced rename */
        refcount_t                ref;                /* Object refcount */
        atomic_t                active;                /* Active user count */
        u32                        addr_version;        /* Address list version */
        u16                        service_id;        /* Service ID we're using. */
        short                        create_error;        /* Creation error */
        unsigned int                rtt;                /* Server's current RTT in uS */
        unsigned int                debug_id;        /* Debugging ID for traces */

        /* file service access */
        rwlock_t                fs_lock;        /* access lock */

        /* Probe state */
        struct afs_endpoint_state __rcu *endpoint_state; /* Latest endpoint/probe state */
        unsigned long                probed_at;        /* Time last probe was dispatched (jiffies) */
        wait_queue_head_t        probe_wq;
        unsigned int                probe_counter;        /* Number of probes issued */
        spinlock_t                probe_lock;
};

enum afs_ro_replicating {
        AFS_RO_NOT_REPLICATING,                        /* Not doing replication */
        AFS_RO_REPLICATING_USE_OLD,                /* Replicating; use old version */
        AFS_RO_REPLICATING_USE_NEW,                /* Replicating; switch to new version */
} __mode(byte);

/*
 * Replaceable volume server list.
 */
struct afs_server_entry {
        struct afs_server        *server;
        struct afs_volume        *volume;
        struct list_head        slink;                /* Link in server->volumes */
        time64_t                cb_expires_at;        /* Time at which volume-level callback expires */
        unsigned long                flags;
#define AFS_SE_EXCLUDED                0                /* Set if server is to be excluded in rotation */
#define AFS_SE_VOLUME_OFFLINE        1                /* Set if volume offline notice given */
#define AFS_SE_VOLUME_BUSY        2                /* Set if volume busy notice given */
};

struct afs_server_list {
        struct rcu_head                rcu;
        refcount_t                usage;
        bool                        attached;        /* T if attached to servers */
        enum afs_ro_replicating        ro_replicating;        /* RW->RO update (probably) in progress */
        unsigned char                nr_servers;
        unsigned short                vnovol_mask;        /* Servers to be skipped due to VNOVOL */
        unsigned int                seq;                /* Set to ->servers_seq when installed */
        rwlock_t                lock;
        struct afs_server_entry        servers[];
};

/*
 * Live AFS volume management.
 */
struct afs_volume {
        struct rcu_head        rcu;
        afs_volid_t                vid;                /* The volume ID of this volume */
        afs_volid_t                vids[AFS_MAXTYPES]; /* All associated volume IDs */
        refcount_t                ref;
        unsigned int                debug_id;        /* Debugging ID for traces */
        time64_t                update_at;        /* Time at which to next update */
        struct afs_cell                *cell;                /* Cell to which belongs (pins ref) */
        struct rb_node                cell_node;        /* Link in cell->volumes */
        struct hlist_node        proc_link;        /* Link in cell->proc_volumes */
        struct super_block __rcu *sb;                /* Superblock on which inodes reside */
        struct work_struct        destructor;        /* Deferred destructor */
        unsigned long                flags;
#define AFS_VOLUME_NEEDS_UPDATE        0        /* - T if an update needs performing */
#define AFS_VOLUME_UPDATING        1        /* - T if an update is in progress */
#define AFS_VOLUME_WAIT                2        /* - T if users must wait for update */
#define AFS_VOLUME_DELETED        3        /* - T if volume appears deleted */
#define AFS_VOLUME_MAYBE_NO_IBULK 4        /* - T if some servers don't have InlineBulkStatus */
#define AFS_VOLUME_RM_TREE        5        /* - Set if volume removed from cell->volumes */
#ifdef CONFIG_AFS_FSCACHE
        struct fscache_volume        *cache;                /* Caching cookie */
#endif
        struct afs_server_list __rcu *servers;        /* List of servers on which volume resides */
        rwlock_t                servers_lock;        /* Lock for ->servers */
        unsigned int                servers_seq;        /* Incremented each time ->servers changes */

        /* RO release tracking */
        struct mutex                volsync_lock;        /* Time/state evaluation lock */
        time64_t                creation_time;        /* Volume creation time (or TIME64_MIN) */
        time64_t                update_time;        /* Volume update time (or TIME64_MIN) */

        /* Callback management */
        struct mutex                cb_check_lock;        /* Lock to control race to check after v_break */
        time64_t                cb_expires_at;        /* Earliest volume callback expiry time */
        atomic_t                cb_ro_snapshot;        /* RO volume update-from-snapshot counter */
        atomic_t                cb_v_break;        /* Volume-break event counter. */
        atomic_t                cb_v_check;        /* Volume-break has-been-checked counter. */
        atomic_t                cb_scrub;        /* Scrub-all-data event counter. */
        rwlock_t                cb_v_break_lock;
        struct rw_semaphore        open_mmaps_lock;
        struct list_head        open_mmaps;        /* List of vnodes that are mmapped */

        afs_voltype_t                type;                /* type of volume */
        char                        type_force;        /* force volume type (suppress R/O -> R/W) */
        u8                        name_len;
        u8                        name[AFS_MAXVOLNAME + 1]; /* NUL-padded volume name */
};

enum afs_lock_state {
        AFS_VNODE_LOCK_NONE,                /* The vnode has no lock on the server */
        AFS_VNODE_LOCK_WAITING_FOR_CB,        /* We're waiting for the server to break the callback */
        AFS_VNODE_LOCK_SETTING,                /* We're asking the server for a lock */
        AFS_VNODE_LOCK_GRANTED,                /* We have a lock on the server */
        AFS_VNODE_LOCK_EXTENDING,        /* We're extending a lock on the server */
        AFS_VNODE_LOCK_NEED_UNLOCK,        /* We need to unlock on the server */
        AFS_VNODE_LOCK_UNLOCKING,        /* We're telling the server to unlock */
        AFS_VNODE_LOCK_DELETED,                /* The vnode has been deleted whilst we have a lock */
};

/*
 * AFS inode private data.
 *
 * Note that afs_alloc_inode() *must* reset anything that could incorrectly
 * leak from one inode to another.
 */
struct afs_vnode {
        struct netfs_inode        netfs;                /* Netfslib context and vfs inode */
        struct afs_volume        *volume;        /* volume on which vnode resides */
        struct afs_fid                fid;                /* the file identifier for this inode */
        struct afs_file_status        status;                /* AFS status info for this file */
        afs_dataversion_t        invalid_before;        /* Child dentries are invalid before this */
        struct afs_permits __rcu *permit_cache;        /* cache of permits so far obtained */
        struct list_head        io_lock_waiters; /* Threads waiting for the I/O lock */
        struct rw_semaphore        validate_lock;        /* lock for validating this vnode */
        struct rw_semaphore        rmdir_lock;        /* Lock for rmdir vs sillyrename */
        struct key                *silly_key;        /* Silly rename key */
        spinlock_t                wb_lock;        /* lock for wb_keys */
        spinlock_t                lock;                /* waitqueue/flags lock */
        unsigned long                flags;
#define AFS_VNODE_IO_LOCK        0                /* Set if the I/O serialisation lock is held */
#define AFS_VNODE_UNSET                1                /* set if vnode attributes not yet set */
#define AFS_VNODE_DIR_VALID        2                /* Set if dir contents are valid */
#define AFS_VNODE_ZAP_DATA        3                /* set if vnode's data should be invalidated */
#define AFS_VNODE_DELETED        4                /* set if vnode deleted on server */
#define AFS_VNODE_MOUNTPOINT        5                /* set if vnode is a mountpoint symlink */
#define AFS_VNODE_PSEUDODIR        7                 /* set if Vnode is a pseudo directory */
#define AFS_VNODE_NEW_CONTENT        8                /* Set if file has new content (create/trunc-0) */
#define AFS_VNODE_SILLY_DELETED        9                /* Set if file has been silly-deleted */
#define AFS_VNODE_MODIFYING        10                /* Set if we're performing a modification op */
#define AFS_VNODE_DIR_READ        11                /* Set if we've read a dir's contents */

        struct folio_queue        *directory;        /* Directory contents */
        struct afs_symlink __rcu *symlink;        /* Symlink content */
        struct list_head        wb_keys;        /* List of keys available for writeback */
        struct list_head        pending_locks;        /* locks waiting to be granted */
        struct list_head        granted_locks;        /* locks granted on this file */
        struct delayed_work        lock_work;        /* work to be done in locking */
        struct key                *lock_key;        /* Key to be used in lock ops */
        ktime_t                        locked_at;        /* Time at which lock obtained */
        enum afs_lock_state        lock_state : 8;
        afs_lock_type_t                lock_type : 8;
        unsigned int                directory_size;        /* Amount of space in ->directory */

        /* outstanding callback notification on this file */
        struct work_struct        cb_work;        /* Work for mmap'd files */
        struct list_head        cb_mmap_link;        /* Link in cell->fs_open_mmaps */
        void                        *cb_server;        /* Server with callback/filelock */
        atomic_t                cb_nr_mmap;        /* Number of mmaps */
        unsigned int                cb_ro_snapshot;        /* RO volume release counter on ->volume */
        unsigned int                cb_scrub;        /* Scrub counter on ->volume */
        unsigned int                cb_break;        /* Break counter on vnode */
        unsigned int                cb_v_check;        /* Break check counter on ->volume */
        seqlock_t                cb_lock;        /* Lock for ->cb_server, ->status, ->cb_*break */

        atomic64_t                cb_expires_at;        /* time at which callback expires */
#define AFS_NO_CB_PROMISE TIME64_MIN
};

static inline struct fscache_cookie *afs_vnode_cache(struct afs_vnode *vnode)
{
#ifdef CONFIG_AFS_FSCACHE
        return netfs_i_cookie(&vnode->netfs);
#else
        return NULL;
#endif
}

static inline void afs_vnode_set_cache(struct afs_vnode *vnode,
                                       struct fscache_cookie *cookie)
{
#ifdef CONFIG_AFS_FSCACHE
        vnode->netfs.cache = cookie;
#endif
}

/*
 * cached security record for one user's attempt to access a vnode
 */
struct afs_permit {
        struct key                *key;                /* RxRPC ticket holding a security context */
        afs_access_t                access;                /* CallerAccess value for this key */
};

/*
 * Immutable cache of CallerAccess records from attempts to access vnodes.
 * These may be shared between multiple vnodes.
 */
struct afs_permits {
        struct rcu_head                rcu;
        struct hlist_node        hash_node;        /* Link in hash */
        unsigned long                h;                /* Hash value for this permit list */
        refcount_t                usage;
        unsigned short                nr_permits;        /* Number of records */
        bool                        invalidated;        /* Invalidated due to key change */
        struct afs_permit        permits[] __counted_by(nr_permits);        /* List of permits sorted by key pointer */
};

/*
 * Copy of symlink content for normal use.
 */
struct afs_symlink {
        struct rcu_head                rcu;
        refcount_t                ref;
        char                        content[];
};

/*
 * Error prioritisation and accumulation.
 */
struct afs_error {
        s32        abort_code;                /* Cumulative abort code */
        short        error;                        /* Cumulative error */
        bool        responded;                /* T if server responded */
        bool        aborted;                /* T if ->error is from an abort */
};

/*
 * Cursor for iterating over a set of volume location servers.
 */
struct afs_vl_cursor {
        struct afs_cell                *cell;                /* The cell we're querying */
        struct afs_vlserver_list *server_list;        /* Current server list (pins ref) */
        struct afs_vlserver        *server;        /* Server on which this resides */
        struct afs_addr_list        *alist;                /* Current address list (pins ref) */
        struct key                *key;                /* Key for the server */
        unsigned long                untried_servers; /* Bitmask of untried servers */
        unsigned long                addr_tried;        /* Tried addresses */
        struct afs_error        cumul_error;        /* Cumulative error */
        unsigned int                debug_id;
        s32                        call_abort_code;
        short                        call_error;        /* Error from single call */
        short                        server_index;        /* Current server */
        signed char                addr_index;        /* Current address */
        unsigned short                flags;
#define AFS_VL_CURSOR_STOP        0x0001                /* Set to cease iteration */
#define AFS_VL_CURSOR_RETRY        0x0002                /* Set to do a retry */
#define AFS_VL_CURSOR_RETRIED        0x0004                /* Set if started a retry */
        short                        nr_iterations;        /* Number of server iterations */
        bool                        call_responded;        /* T if the current address responded */
};

/*
 * Fileserver state tracking for an operation.  An array of these is kept,
 * indexed by server index.
 */
struct afs_server_state {
        /* Tracking of fileserver probe state.  Other operations may interfere
         * by probing a fileserver when accessing other volumes.
         */
        unsigned int                probe_seq;
        unsigned long                untried_addrs;        /* Addresses we haven't tried yet */
        struct wait_queue_entry        probe_waiter;
        struct afs_endpoint_state *endpoint_state; /* Endpoint state being monitored */
};

/*
 * Fileserver operation methods.
 */
struct afs_operation_ops {
        void (*issue_afs_rpc)(struct afs_operation *op);
        void (*issue_yfs_rpc)(struct afs_operation *op);
        void (*success)(struct afs_operation *op);
        void (*aborted)(struct afs_operation *op);
        void (*failed)(struct afs_operation *op);
        void (*edit_dir)(struct afs_operation *op);
        void (*put)(struct afs_operation *op);
};

struct afs_vnode_param {
        struct afs_vnode        *vnode;
        struct afs_fid                fid;                /* Fid to access */
        struct afs_status_cb        scb;                /* Returned status and callback promise */
        afs_dataversion_t        dv_before;        /* Data version before the call */
        unsigned int                cb_break_before; /* cb_break before the call */
        u8                        dv_delta;        /* Expected change in data version */
        bool                        put_vnode:1;        /* T if we have a ref on the vnode */
        bool                        need_io_lock:1;        /* T if we need the I/O lock on this */
        bool                        update_ctime:1;        /* Need to update the ctime */
        bool                        set_size:1;        /* Must update i_size */
        bool                        op_unlinked:1;        /* True if file was unlinked by op */
        bool                        speculative:1;        /* T if speculative status fetch (no vnode lock) */
        bool                        modification:1;        /* Set if the content gets modified */
};

/*
 * Fileserver operation wrapper, handling server and address rotation
 * asynchronously.  May make simultaneous calls to multiple servers.
 */
struct afs_operation {
        struct afs_net                *net;                /* Network namespace */
        struct key                *key;                /* Key for the cell */
        const struct afs_call_type *type;        /* Type of call done */
        const struct afs_operation_ops *ops;

        /* Parameters/results for the operation */
        struct afs_volume        *volume;        /* Volume being accessed */
        struct afs_vnode_param        file[2];
        struct afs_vnode_param        *more_files;
        struct afs_volsync        pre_volsync;        /* Volsync before op */
        struct afs_volsync        volsync;        /* Volsync returned by op */
        struct dentry                *dentry;        /* Dentry to be altered */
        struct dentry                *dentry_2;        /* Second dentry to be altered */
        struct timespec64        mtime;                /* Modification time to record */
        struct timespec64        ctime;                /* Change time to set */
        struct afs_error        cumul_error;        /* Cumulative error */
        short                        nr_files;        /* Number of entries in file[], more_files */
        unsigned int                debug_id;

        unsigned int                cb_v_break;        /* Volume break counter before op */

        union {
                struct {
                        int        which;                /* Which ->file[] to fetch for */
                } fetch_status;
                struct {
                        int        reason;                /* enum afs_edit_dir_reason */
                        mode_t        mode;
                        struct afs_symlink *symlink;
                } create;
                struct {
                        bool        need_rehash;
                } unlink;
                struct {
                        struct dentry        *rehash;
                        struct dentry        *tmp;
                        unsigned int        rename_flags;
                        bool                new_negative;
                } rename;
                struct {
                        struct netfs_io_subrequest *subreq;
                } fetch;
                struct {
                        afs_lock_type_t type;
                } lock;
                struct {
                        struct iov_iter        *write_iter;
                        loff_t        pos;
                        loff_t        size;
                        loff_t        i_size;
                } store;
                struct {
                        struct iattr        *attr;
                        loff_t                old_i_size;
                } setattr;
                struct afs_acl        *acl;
                struct yfs_acl        *yacl;
                struct {
                        struct afs_volume_status vs;
                        struct kstatfs                *buf;
                } volstatus;
        };

        /* Fileserver iteration state */
        struct afs_server_list        *server_list;        /* Current server list (pins ref) */
        struct afs_server        *server;        /* Server we're using (ref pinned by server_list) */
        struct afs_endpoint_state *estate;        /* Current endpoint state (doesn't pin ref) */
        struct afs_server_state        *server_states;        /* States of the servers involved */
        struct afs_call                *call;
        unsigned long                untried_servers; /* Bitmask of untried servers */
        unsigned long                addr_tried;        /* Tried addresses */
        s32                        call_abort_code; /* Abort code from single call */
        short                        call_error;        /* Error from single call */
        short                        server_index;        /* Current server */
        short                        nr_iterations;        /* Number of server iterations */
        signed char                addr_index;        /* Current address */
        bool                        call_responded;        /* T if the current address responded */

        unsigned int                flags;
#define AFS_OPERATION_STOP                0x0001        /* Set to cease iteration */
#define AFS_OPERATION_VBUSY                0x0002        /* Set if seen VBUSY */
#define AFS_OPERATION_VMOVED                0x0004        /* Set if seen VMOVED */
#define AFS_OPERATION_VNOVOL                0x0008        /* Set if seen VNOVOL */
#define AFS_OPERATION_CUR_ONLY                0x0010        /* Set if current server only (file lock held) */
#define AFS_OPERATION_NO_VSLEEP                0x0020        /* Set to prevent sleep on VBUSY, VOFFLINE, ... */
#define AFS_OPERATION_UNINTR                0x0040        /* Set if op is uninterruptible */
#define AFS_OPERATION_DOWNGRADE                0x0080        /* Set to retry with downgraded opcode */
#define AFS_OPERATION_LOCK_0                0x0100        /* Set if have io_lock on file[0] */
#define AFS_OPERATION_LOCK_1                0x0200        /* Set if have io_lock on file[1] */
#define AFS_OPERATION_TRIED_ALL                0x0400        /* Set if we've tried all the fileservers */
#define AFS_OPERATION_RETRY_SERVER        0x0800        /* Set if we should retry the current server */
#define AFS_OPERATION_DIR_CONFLICT        0x1000        /* Set if we detected a 3rd-party dir change */
#define AFS_OPERATION_ASYNC                0x2000        /* Set if should run asynchronously */
};

/*
 * Cache auxiliary data.
 */
struct afs_vnode_cache_aux {
        __be64                        data_version;
} __packed;

static inline void afs_set_cache_aux(struct afs_vnode *vnode,
                                     struct afs_vnode_cache_aux *aux)
{
        aux->data_version = cpu_to_be64(vnode->status.data_version);
}

static inline void afs_invalidate_cache(struct afs_vnode *vnode, unsigned int flags)
{
        struct afs_vnode_cache_aux aux;

        afs_set_cache_aux(vnode, &aux);
        fscache_invalidate(afs_vnode_cache(vnode), &aux,
                           i_size_read(&vnode->netfs.inode), flags);
}

/*
 * Directory iteration management.
 */
struct afs_dir_iter {
        struct afs_vnode        *dvnode;
        union afs_xdr_dir_block *block;
        struct folio_queue        *fq;
        unsigned int                fpos;
        int                        fq_slot;
        unsigned int                loop_check;
        u8                        nr_slots;
        u8                        bucket;
        unsigned int                prev_entry;
};

#include <trace/events/afs.h>

/*****************************************************************************/
/*
 * addr_list.c
 */
struct afs_addr_list *afs_get_addrlist(struct afs_addr_list *alist, enum afs_alist_trace reason);
extern struct afs_addr_list *afs_alloc_addrlist(unsigned int nr);
extern void afs_put_addrlist(struct afs_addr_list *alist, enum afs_alist_trace reason);
extern struct afs_vlserver_list *afs_parse_text_addrs(struct afs_net *,
                                                      const char *, size_t, char,
                                                      unsigned short, unsigned short);
bool afs_addr_list_same(const struct afs_addr_list *a,
                        const struct afs_addr_list *b);
extern struct afs_vlserver_list *afs_dns_query(struct afs_cell *, time64_t *);

extern int afs_merge_fs_addr4(struct afs_net *net, struct afs_addr_list *addr,
                              __be32 xdr, u16 port);
extern int afs_merge_fs_addr6(struct afs_net *net, struct afs_addr_list *addr,
                              __be32 *xdr, u16 port);
void afs_set_peer_appdata(struct afs_server *server,
                          struct afs_addr_list *old_alist,
                          struct afs_addr_list *new_alist);

/*
 * addr_prefs.c
 */
int afs_proc_addr_prefs_write(struct file *file, char *buf, size_t size);
void afs_get_address_preferences_rcu(struct afs_net *net, struct afs_addr_list *alist);
void afs_get_address_preferences(struct afs_net *net, struct afs_addr_list *alist);

/*
 * callback.c
 */
extern void afs_invalidate_mmap_work(struct work_struct *);
extern void afs_init_callback_state(struct afs_server *);
extern void __afs_break_callback(struct afs_vnode *, enum afs_cb_break_reason);
extern void afs_break_callback(struct afs_vnode *, enum afs_cb_break_reason);
extern void afs_break_callbacks(struct afs_server *, size_t, struct afs_callback_break *);

static inline unsigned int afs_calc_vnode_cb_break(struct afs_vnode *vnode)
{
        return vnode->cb_break + vnode->cb_ro_snapshot + vnode->cb_scrub;
}

static inline bool afs_cb_is_broken(unsigned int cb_break,
                                    const struct afs_vnode *vnode)
{
        return cb_break != (vnode->cb_break +
                            atomic_read(&vnode->volume->cb_ro_snapshot) +
                            atomic_read(&vnode->volume->cb_scrub));
}

/*
 * cell.c
 */
extern int afs_cell_init(struct afs_net *, const char *);
extern struct afs_cell *afs_find_cell(struct afs_net *, const char *, unsigned,
                                      enum afs_cell_trace);
enum afs_lookup_cell_for {
        AFS_LOOKUP_CELL_DYNROOT,
        AFS_LOOKUP_CELL_MOUNTPOINT,
        AFS_LOOKUP_CELL_DIRECT_MOUNT,
        AFS_LOOKUP_CELL_PRELOAD,
        AFS_LOOKUP_CELL_ROOTCELL,
        AFS_LOOKUP_CELL_ALIAS_CHECK,
};
struct afs_cell *afs_lookup_cell(struct afs_net *net,
                                 const char *name, unsigned int namesz,
                                 const char *vllist,
                                 enum afs_lookup_cell_for reason,
                                 enum afs_cell_trace trace);
extern struct afs_cell *afs_use_cell(struct afs_cell *, enum afs_cell_trace);
void afs_unuse_cell(struct afs_cell *cell, enum afs_cell_trace reason);
extern struct afs_cell *afs_get_cell(struct afs_cell *, enum afs_cell_trace);
extern void afs_see_cell(struct afs_cell *, enum afs_cell_trace);
extern void afs_put_cell(struct afs_cell *, enum afs_cell_trace);
extern void afs_queue_cell(struct afs_cell *, enum afs_cell_trace);
void afs_set_cell_timer(struct afs_cell *cell, unsigned int delay_secs);
extern void __net_exit afs_cell_purge(struct afs_net *);

/*
 * cmservice.c
 */
extern bool afs_cm_incoming_call(struct afs_call *);

/*
 * cm_security.c
 */
void afs_process_oob_queue(struct work_struct *work);
#ifdef CONFIG_RXGK
int afs_create_token_key(struct afs_net *net, struct socket *socket);
#else
static inline int afs_create_token_key(struct afs_net *net, struct socket *socket)
{
        return 0;
}
#endif

/*
 * dir.c
 */
extern const struct file_operations afs_dir_file_operations;
extern const struct inode_operations afs_dir_inode_operations;
extern const struct address_space_operations afs_dir_aops;
extern const struct dentry_operations afs_fs_dentry_operations;

ssize_t afs_read_dir(struct afs_vnode *dvnode, struct file *file)
        __acquires(&dvnode->validate_lock);
extern void afs_d_release(struct dentry *);
extern void afs_check_for_remote_deletion(struct afs_operation *);

/*
 * dir_edit.c
 */
extern void afs_edit_dir_add(struct afs_vnode *, const struct qstr *, struct afs_fid *,
                             enum afs_edit_dir_reason);
extern void afs_edit_dir_remove(struct afs_vnode *, const struct qstr *, enum afs_edit_dir_reason);
void afs_edit_dir_update(struct afs_vnode *vnode, const struct qstr *name,
                         struct afs_vnode *new_dvnode, enum afs_edit_dir_reason why);
void afs_mkdir_init_dir(struct afs_vnode *dvnode, struct afs_vnode *parent_vnode);

/*
 * dir_search.c
 */
unsigned int afs_dir_hash_name(const struct qstr *name);
bool afs_dir_init_iter(struct afs_dir_iter *iter, const struct qstr *name);
union afs_xdr_dir_block *afs_dir_find_block(struct afs_dir_iter *iter, size_t block);
int afs_dir_search_bucket(struct afs_dir_iter *iter, const struct qstr *name,
                          struct afs_fid *_fid);
int afs_dir_search(struct afs_vnode *dvnode, const struct qstr *name,
                   struct afs_fid *_fid, afs_dataversion_t *_dir_version);

/*
 * dir_silly.c
 */
extern int afs_sillyrename(struct afs_vnode *, struct afs_vnode *,
                           struct dentry *, struct key *);
extern int afs_silly_iput(struct dentry *, struct inode *);

/*
 * dynroot.c
 */
extern const struct inode_operations afs_dynroot_inode_operations;
extern const struct dentry_operations afs_dynroot_dentry_operations;

struct inode *afs_dynroot_iget_root(struct super_block *sb);

/*
 * file.c
 */
extern const struct address_space_operations afs_file_aops;
extern const struct inode_operations afs_file_inode_operations;
extern const struct file_operations afs_file_operations;
extern const struct afs_operation_ops afs_fetch_data_operation;
extern const struct netfs_request_ops afs_req_ops;

extern int afs_cache_wb_key(struct afs_vnode *, struct afs_file *);
extern void afs_put_wb_key(struct afs_wb_key *);
extern int afs_open(struct inode *, struct file *);
extern int afs_release(struct inode *, struct file *);
void afs_fetch_data_async_rx(struct work_struct *work);
void afs_fetch_data_immediate_cancel(struct afs_call *call);
void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size);

/*
 * flock.c
 */
extern struct workqueue_struct *afs_lock_manager;

extern void afs_lock_op_done(struct afs_call *);
extern void afs_lock_work(struct work_struct *);
extern void afs_lock_may_be_available(struct afs_vnode *);
extern int afs_lock(struct file *, int, struct file_lock *);
extern int afs_flock(struct file *, int, struct file_lock *);

/*
 * fsclient.c
 */
extern void afs_fs_fetch_status(struct afs_operation *);
extern void afs_fs_fetch_data(struct afs_operation *);
extern void afs_fs_create_file(struct afs_operation *);
extern void afs_fs_make_dir(struct afs_operation *);
extern void afs_fs_remove_file(struct afs_operation *);
extern void afs_fs_remove_dir(struct afs_operation *);
extern void afs_fs_link(struct afs_operation *);
extern void afs_fs_symlink(struct afs_operation *);
extern void afs_fs_rename(struct afs_operation *);
extern void afs_fs_store_data(struct afs_operation *);
extern void afs_fs_setattr(struct afs_operation *);
extern void afs_fs_get_volume_status(struct afs_operation *);
extern void afs_fs_set_lock(struct afs_operation *);
extern void afs_fs_extend_lock(struct afs_operation *);
extern void afs_fs_release_lock(struct afs_operation *);
int afs_fs_give_up_all_callbacks(struct afs_net *net, struct afs_server *server,
                                 struct afs_address *addr, struct key *key);
bool afs_fs_get_capabilities(struct afs_net *net, struct afs_server *server,
                             struct afs_endpoint_state *estate, unsigned int addr_index,
                             struct key *key);
extern void afs_fs_inline_bulk_status(struct afs_operation *);

struct afs_acl {
        u32        size;
        u8        data[] __counted_by(size);
};

extern void afs_fs_fetch_acl(struct afs_operation *);
extern void afs_fs_store_acl(struct afs_operation *);

/*
 * fs_operation.c
 */
extern struct afs_operation *afs_alloc_operation(struct key *, struct afs_volume *);
extern int afs_put_operation(struct afs_operation *);
extern bool afs_begin_vnode_operation(struct afs_operation *);
extern void afs_end_vnode_operation(struct afs_operation *op);
extern void afs_wait_for_operation(struct afs_operation *);
extern int afs_do_sync_operation(struct afs_operation *);

static inline void afs_op_set_vnode(struct afs_operation *op, unsigned int n,
                                    struct afs_vnode *vnode)
{
        op->file[n].vnode = vnode;
        op->file[n].need_io_lock = true;
}

static inline void afs_op_set_fid(struct afs_operation *op, unsigned int n,
                                  const struct afs_fid *fid)
{
        op->file[n].fid = *fid;
}

/*
 * fs_probe.c
 */
struct afs_endpoint_state *afs_get_endpoint_state(struct afs_endpoint_state *estate,
                                                  enum afs_estate_trace where);
void afs_put_endpoint_state(struct afs_endpoint_state *estate, enum afs_estate_trace where);
extern void afs_fileserver_probe_result(struct afs_call *);
int afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server,
                            struct afs_addr_list *new_alist, struct key *key);
int afs_wait_for_fs_probes(struct afs_operation *op, struct afs_server_state *states, bool intr);
extern void afs_probe_fileserver(struct afs_net *, struct afs_server *);
extern void afs_fs_probe_dispatcher(struct work_struct *);
int afs_wait_for_one_fs_probe(struct afs_server *server, struct afs_endpoint_state *estate,
                              unsigned long exclude, bool is_intr);
extern void afs_fs_probe_cleanup(struct afs_net *);

/*
 * inode.c
 */
extern const struct afs_operation_ops afs_fetch_status_operation;

extern void afs_vnode_commit_status(struct afs_operation *, struct afs_vnode_param *);
extern int afs_fetch_status(struct afs_vnode *, struct key *, bool, afs_access_t *);
extern int afs_ilookup5_test_by_fid(struct inode *, void *);
extern struct inode *afs_iget(struct afs_operation *, struct afs_vnode_param *);
extern struct inode *afs_root_iget(struct super_block *, struct key *);
extern int afs_getattr(struct mnt_idmap *idmap, const struct path *,
                       struct kstat *, u32, unsigned int);
extern int afs_setattr(struct mnt_idmap *idmap, struct dentry *, struct iattr *);
extern void afs_evict_inode(struct inode *);
extern int afs_drop_inode(struct inode *);

/*
 * main.c
 */
extern struct workqueue_struct *afs_wq;
extern int afs_net_id;

static inline struct afs_net *afs_net(struct net *net)
{
        return net_generic(net, afs_net_id);
}

static inline struct afs_net *afs_sb2net(struct super_block *sb)
{
        return afs_net(AFS_FS_S(sb)->net_ns);
}

static inline struct afs_net *afs_d2net(struct dentry *dentry)
{
        return afs_sb2net(dentry->d_sb);
}

static inline struct afs_net *afs_i2net(struct inode *inode)
{
        return afs_sb2net(inode->i_sb);
}

static inline struct afs_net *afs_v2net(struct afs_vnode *vnode)
{
        return afs_i2net(&vnode->netfs.inode);
}

static inline struct afs_net *afs_sock2net(struct sock *sk)
{
        return net_generic(sock_net(sk), afs_net_id);
}

static inline void __afs_stat(atomic_t *s)
{
        atomic_inc(s);
}

#define afs_stat_v(vnode, n) __afs_stat(&afs_v2net(vnode)->n)

/*
 * misc.c
 */
extern int afs_abort_to_error(u32);
extern void afs_prioritise_error(struct afs_error *, int, u32);

static inline void afs_op_nomem(struct afs_operation *op)
{
        op->cumul_error.error = -ENOMEM;
}

static inline int afs_op_error(const struct afs_operation *op)
{
        return op->cumul_error.error;
}

static inline s32 afs_op_abort_code(const struct afs_operation *op)
{
        return op->cumul_error.abort_code;
}

static inline int afs_op_set_error(struct afs_operation *op, int error)
{
        return op->cumul_error.error = error;
}

static inline void afs_op_accumulate_error(struct afs_operation *op, int error, s32 abort_code)
{
        afs_prioritise_error(&op->cumul_error, error, abort_code);
}

/*
 * mntpt.c
 */
extern const struct inode_operations afs_mntpt_inode_operations;
extern const struct inode_operations afs_autocell_inode_operations;
extern const struct file_operations afs_mntpt_file_operations;

extern struct vfsmount *afs_d_automount(struct path *);
extern void afs_mntpt_kill_timer(void);

/*
 * proc.c
 */
#ifdef CONFIG_PROC_FS
extern int __net_init afs_proc_init(struct afs_net *);
extern void __net_exit afs_proc_cleanup(struct afs_net *);
extern int afs_proc_cell_setup(struct afs_cell *);
extern void afs_proc_cell_remove(struct afs_cell *);
extern void afs_put_sysnames(struct afs_sysnames *);
#else
static inline int afs_proc_init(struct afs_net *net) { return 0; }
static inline void afs_proc_cleanup(struct afs_net *net) {}
static inline int afs_proc_cell_setup(struct afs_cell *cell) { return 0; }
static inline void afs_proc_cell_remove(struct afs_cell *cell) {}
static inline void afs_put_sysnames(struct afs_sysnames *sysnames) {}
#endif

/*
 * rotate.c
 */
void afs_clear_server_states(struct afs_operation *op);
extern bool afs_select_fileserver(struct afs_operation *);
extern void afs_dump_edestaddrreq(const struct afs_operation *);

/*
 * rxrpc.c
 */
extern struct workqueue_struct *afs_async_calls;

extern int __net_init afs_open_socket(struct afs_net *);
extern void __net_exit afs_close_socket(struct afs_net *);
extern void afs_charge_preallocation(struct work_struct *);
extern void afs_put_call(struct afs_call *);
void afs_deferred_put_call(struct afs_call *call);
void afs_make_call(struct afs_call *call, gfp_t gfp);
void afs_deliver_to_call(struct afs_call *call);
void afs_wait_for_call_to_complete(struct afs_call *call);
extern struct afs_call *afs_alloc_flat_call(struct afs_net *,
                                            const struct afs_call_type *,
                                            size_t, size_t);
extern void afs_flat_call_destructor(struct afs_call *);
extern void afs_send_empty_reply(struct afs_call *);
extern void afs_send_simple_reply(struct afs_call *, const void *, size_t);
extern int afs_extract_data(struct afs_call *, bool);
extern int afs_protocol_error(struct afs_call *, enum afs_eproto_cause);

static inline struct afs_call *afs_get_call(struct afs_call *call,
                                            enum afs_call_trace why)
{
        int r;

        __refcount_inc(&call->ref, &r);

        trace_afs_call(call->debug_id, why, r + 1,
                       atomic_read(&call->net->nr_outstanding_calls),
                       __builtin_return_address(0));
        return call;
}

static inline void afs_see_call(struct afs_call *call, enum afs_call_trace why)
{
        int r = refcount_read(&call->ref);

        trace_afs_call(call->debug_id, why, r,
                       atomic_read(&call->net->nr_outstanding_calls),
                       __builtin_return_address(0));
}

static inline void afs_make_op_call(struct afs_operation *op, struct afs_call *call,
                                    gfp_t gfp)
{
        struct afs_addr_list *alist = op->estate->addresses;

        op->call        = afs_get_call(call, afs_call_trace_get);
        op->type        = call->type;
        call->op        = op;
        call->key        = op->key;
        call->intr        = !(op->flags & AFS_OPERATION_UNINTR);
        call->peer        = rxrpc_kernel_get_peer(alist->addrs[op->addr_index].peer);
        call->service_id = op->server->service_id;
        afs_make_call(call, gfp);
        afs_put_call(call);
}

static inline void afs_extract_begin(struct afs_call *call, void *buf, size_t size)
{
        call->iov_len = size;
        call->kvec[0].iov_base = buf;
        call->kvec[0].iov_len = size;
        iov_iter_kvec(&call->def_iter, ITER_DEST, call->kvec, 1, size);
}

static inline void afs_extract_to_tmp(struct afs_call *call)
{
        call->iov_len = sizeof(call->tmp);
        afs_extract_begin(call, &call->tmp, sizeof(call->tmp));
}

static inline void afs_extract_to_tmp64(struct afs_call *call)
{
        call->iov_len = sizeof(call->tmp64);
        afs_extract_begin(call, &call->tmp64, sizeof(call->tmp64));
}

static inline void afs_extract_discard(struct afs_call *call, size_t size)
{
        call->iov_len = size;
        iov_iter_discard(&call->def_iter, ITER_DEST, size);
}

static inline void afs_extract_to_buf(struct afs_call *call, size_t size)
{
        call->iov_len = size;
        afs_extract_begin(call, call->buffer, size);
}

static inline int afs_transfer_reply(struct afs_call *call)
{
        return afs_extract_data(call, false);
}

static inline bool afs_check_call_state(struct afs_call *call,
                                        enum afs_call_state state)
{
        return READ_ONCE(call->state) == state;
}

static inline bool afs_set_call_state(struct afs_call *call,
                                      enum afs_call_state from,
                                      enum afs_call_state to)
{
        bool ok = false;

        spin_lock_bh(&call->state_lock);
        if (call->state == from) {
                call->state = to;
                trace_afs_call_state(call, from, to, 0, 0);
                ok = true;
        }
        spin_unlock_bh(&call->state_lock);
        return ok;
}

static inline void afs_set_call_complete(struct afs_call *call,
                                         int error, u32 remote_abort)
{
        enum afs_call_state state;
        bool ok = false;

        spin_lock_bh(&call->state_lock);
        state = call->state;
        if (state != AFS_CALL_COMPLETE) {
                call->abort_code = remote_abort;
                call->error = error;
                call->state = AFS_CALL_COMPLETE;
                trace_afs_call_state(call, state, AFS_CALL_COMPLETE,
                                     error, remote_abort);
                ok = true;
        }
        spin_unlock_bh(&call->state_lock);
        if (ok) {
                trace_afs_call_done(call);

                /* Asynchronous calls have two refs to release - one from the alloc and
                 * one queued with the work item - and we can't just deallocate the
                 * call because the work item may be queued again.
                 */
                if (call->drop_ref)
                        afs_put_call(call);
        }
}

/*
 * security.c
 */
extern void afs_put_permits(struct afs_permits *);
extern void afs_clear_permits(struct afs_vnode *);
extern void afs_cache_permit(struct afs_vnode *, struct key *, unsigned int,
                             struct afs_status_cb *);
extern struct key *afs_request_key(struct afs_cell *);
extern struct key *afs_request_key_rcu(struct afs_cell *);
extern int afs_check_permit(struct afs_vnode *, struct key *, afs_access_t *);
extern int afs_permission(struct mnt_idmap *, struct inode *, int);
extern void __exit afs_clean_up_permit_cache(void);

/*
 * server.c
 */
extern spinlock_t afs_server_peer_lock;

struct afs_server *afs_find_server(const struct rxrpc_peer *peer);
extern struct afs_server *afs_lookup_server(struct afs_cell *, struct key *, const uuid_t *, u32);
extern struct afs_server *afs_get_server(struct afs_server *, enum afs_server_trace);
struct afs_server *afs_use_server(struct afs_server *server, bool activate,
                                  enum afs_server_trace reason);
void afs_unuse_server(struct afs_net *net, struct afs_server *server,
                      enum afs_server_trace reason);
void afs_unuse_server_notime(struct afs_net *net, struct afs_server *server,
                             enum afs_server_trace reason);
extern void afs_put_server(struct afs_net *, struct afs_server *, enum afs_server_trace);
void afs_purge_servers(struct afs_cell *cell);
extern void afs_fs_probe_timer(struct timer_list *);
void __net_exit afs_wait_for_servers(struct afs_net *net);
bool afs_check_server_record(struct afs_operation *op, struct afs_server *server, struct key *key);

static inline void afs_see_server(struct afs_server *server, enum afs_server_trace trace)
{
        int r = refcount_read(&server->ref);
        int a = atomic_read(&server->active);

        trace_afs_server(server->debug_id, r, a, trace);

}

static inline void afs_inc_servers_outstanding(struct afs_net *net)
{
        atomic_inc(&net->servers_outstanding);
}

static inline void afs_dec_servers_outstanding(struct afs_net *net)
{
        if (atomic_dec_and_test(&net->servers_outstanding))
                wake_up_var(&net->servers_outstanding);
}

static inline bool afs_is_probing_server(struct afs_server *server)
{
        return list_empty(&server->probe_link);
}

/*
 * server_list.c
 */
static inline struct afs_server_list *afs_get_serverlist(struct afs_server_list *slist)
{
        refcount_inc(&slist->usage);
        return slist;
}

extern void afs_put_serverlist(struct afs_net *, struct afs_server_list *);
struct afs_server_list *afs_alloc_server_list(struct afs_volume *volume,
                                              struct key *key,
                                              struct afs_vldb_entry *vldb);
extern bool afs_annotate_server_list(struct afs_server_list *, struct afs_server_list *);
void afs_attach_volume_to_servers(struct afs_volume *volume, struct afs_server_list *slist);
void afs_reattach_volume_to_servers(struct afs_volume *volume, struct afs_server_list *slist,
                                    struct afs_server_list *old);
void afs_detach_volume_from_servers(struct afs_volume *volume, struct afs_server_list *slist);

/*
 * super.c
 */
extern int __init afs_fs_init(void);
extern void afs_fs_exit(void);

/*
 * symlink.c
 */
extern const struct inode_operations afs_symlink_inode_operations;
extern const struct address_space_operations afs_symlink_aops;

void afs_invalidate_symlink(struct afs_vnode *vnode);
void afs_evict_symlink(struct afs_vnode *vnode);
void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op);
const char *afs_get_link(struct dentry *dentry, struct inode *inode,
                         struct delayed_call *callback);
int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen);
int afs_symlink_writepages(struct address_space *mapping,
                           struct writeback_control *wbc);

/*
 * validation.c
 */
bool afs_check_validity(const struct afs_vnode *vnode);
int afs_update_volume_state(struct afs_operation *op);
int afs_validate(struct afs_vnode *vnode, struct key *key);

/*
 * vlclient.c
 */
extern struct afs_vldb_entry *afs_vl_get_entry_by_name_u(struct afs_vl_cursor *,
                                                         const char *, int);
extern struct afs_addr_list *afs_vl_get_addrs_u(struct afs_vl_cursor *, const uuid_t *);
struct afs_call *afs_vl_get_capabilities(struct afs_net *net,
                                         struct afs_addr_list *alist,
                                         unsigned int addr_index,
                                         struct key *key,
                                         struct afs_vlserver *server,
                                         unsigned int server_index);
extern struct afs_addr_list *afs_yfsvl_get_endpoints(struct afs_vl_cursor *, const uuid_t *);
extern char *afs_yfsvl_get_cell_name(struct afs_vl_cursor *);

/*
 * vl_alias.c
 */
extern int afs_cell_detect_alias(struct afs_cell *, struct key *);

/*
 * vl_probe.c
 */
extern void afs_vlserver_probe_result(struct afs_call *);
extern int afs_send_vl_probes(struct afs_net *, struct key *, struct afs_vlserver_list *);
extern int afs_wait_for_vl_probes(struct afs_vlserver_list *, unsigned long);

/*
 * vl_rotate.c
 */
extern bool afs_begin_vlserver_operation(struct afs_vl_cursor *,
                                         struct afs_cell *, struct key *);
extern bool afs_select_vlserver(struct afs_vl_cursor *);
extern bool afs_select_current_vlserver(struct afs_vl_cursor *);
extern int afs_end_vlserver_operation(struct afs_vl_cursor *);

/*
 * vlserver_list.c
 */
static inline struct afs_vlserver *afs_get_vlserver(struct afs_vlserver *vlserver)
{
        refcount_inc(&vlserver->ref);
        return vlserver;
}

static inline struct afs_vlserver_list *afs_get_vlserverlist(struct afs_vlserver_list *vllist)
{
        if (vllist)
                refcount_inc(&vllist->ref);
        return vllist;
}

extern struct afs_vlserver *afs_alloc_vlserver(const char *, size_t, unsigned short);
extern void afs_put_vlserver(struct afs_net *, struct afs_vlserver *);
extern struct afs_vlserver_list *afs_alloc_vlserver_list(unsigned int);
extern void afs_put_vlserverlist(struct afs_net *, struct afs_vlserver_list *);
extern struct afs_vlserver_list *afs_extract_vlserver_list(struct afs_cell *,
                                                           const void *, size_t);

/*
 * volume.c
 */
extern struct afs_volume *afs_create_volume(struct afs_fs_context *);
extern int afs_activate_volume(struct afs_volume *);
extern void afs_deactivate_volume(struct afs_volume *);
bool afs_try_get_volume(struct afs_volume *volume, enum afs_volume_trace reason);
extern struct afs_volume *afs_get_volume(struct afs_volume *, enum afs_volume_trace);
void afs_put_volume(struct afs_volume *volume, enum afs_volume_trace reason);
extern int afs_check_volume_status(struct afs_volume *, struct afs_operation *);

/*
 * write.c
 */
void afs_prepare_write(struct netfs_io_subrequest *subreq);
void afs_issue_write(struct netfs_io_subrequest *subreq);
void afs_begin_writeback(struct netfs_io_request *wreq);
void afs_retry_request(struct netfs_io_request *wreq, struct netfs_io_stream *stream);
extern int afs_writepages(struct address_space *, struct writeback_control *);
extern int afs_fsync(struct file *, loff_t, loff_t, int);
extern vm_fault_t afs_page_mkwrite(struct vm_fault *vmf);
extern void afs_prune_wb_keys(struct afs_vnode *);

/*
 * xattr.c
 */
extern const struct xattr_handler * const afs_xattr_handlers[];

/*
 * yfsclient.c
 */
extern void yfs_fs_fetch_data(struct afs_operation *);
extern void yfs_fs_create_file(struct afs_operation *);
extern void yfs_fs_make_dir(struct afs_operation *);
extern void yfs_fs_remove_file2(struct afs_operation *);
extern void yfs_fs_remove_file(struct afs_operation *);
extern void yfs_fs_remove_dir(struct afs_operation *);
extern void yfs_fs_link(struct afs_operation *);
extern void yfs_fs_symlink(struct afs_operation *);
extern void yfs_fs_rename(struct afs_operation *);
void yfs_fs_rename_replace(struct afs_operation *op);
void yfs_fs_rename_noreplace(struct afs_operation *op);
void yfs_fs_rename_exchange(struct afs_operation *op);
extern void yfs_fs_store_data(struct afs_operation *);
extern void yfs_fs_setattr(struct afs_operation *);
extern void yfs_fs_get_volume_status(struct afs_operation *);
extern void yfs_fs_set_lock(struct afs_operation *);
extern void yfs_fs_extend_lock(struct afs_operation *);
extern void yfs_fs_release_lock(struct afs_operation *);
extern void yfs_fs_fetch_status(struct afs_operation *);
extern void yfs_fs_inline_bulk_status(struct afs_operation *);

struct yfs_acl {
        struct afs_acl        *acl;                /* Dir/file/symlink ACL */
        struct afs_acl        *vol_acl;        /* Whole volume ACL */
        u32                inherit_flag;        /* True if ACL is inherited from parent dir */
        u32                num_cleaned;        /* Number of ACEs removed due to subject removal */
        unsigned int        flags;
#define YFS_ACL_WANT_ACL        0x01        /* Set if caller wants ->acl */
#define YFS_ACL_WANT_VOL_ACL        0x02        /* Set if caller wants ->vol_acl */
};

extern void yfs_free_opaque_acl(struct yfs_acl *);
extern void yfs_fs_fetch_opaque_acl(struct afs_operation *);
extern void yfs_fs_store_opaque_acl2(struct afs_operation *);

/*
 * Miscellaneous inline functions.
 */
static inline struct afs_vnode *AFS_FS_I(struct inode *inode)
{
        return container_of(inode, struct afs_vnode, netfs.inode);
}

static inline struct inode *AFS_VNODE_TO_I(struct afs_vnode *vnode)
{
        return &vnode->netfs.inode;
}

/*
 * Note that a dentry got changed.  We need to set d_fsdata to the data version
 * number derived from the result of the operation.  It doesn't matter if
 * d_fsdata goes backwards as we'll just revalidate.
 */
static inline void afs_update_dentry_version(struct afs_operation *op,
                                             struct afs_vnode_param *dir_vp,
                                             struct dentry *dentry)
{
        if (!op->cumul_error.error)
                dentry->d_fsdata =
                        (void *)(unsigned long)dir_vp->scb.status.data_version;
}

/*
 * Check for a conflicting operation on a directory that we just unlinked from.
 * If someone managed to sneak a link or an unlink in on the file we just
 * unlinked, we won't be able to trust nlink on an AFS file (but not YFS).
 */
static inline void afs_check_dir_conflict(struct afs_operation *op,
                                          struct afs_vnode_param *dvp)
{
        if (dvp->dv_before + dvp->dv_delta != dvp->scb.status.data_version)
                op->flags |= AFS_OPERATION_DIR_CONFLICT;
}

static inline int afs_io_error(struct afs_call *call, enum afs_io_error where)
{
        trace_afs_io_error(call->debug_id, -EIO, where);
        return -EIO;
}

static inline int afs_bad(struct afs_vnode *vnode, enum afs_file_error where)
{
        trace_afs_file_error(vnode, -EIO, where);
        return -EIO;
}

/*
 * Set the callback promise on a vnode.
 */
static inline void afs_set_cb_promise(struct afs_vnode *vnode, time64_t expires_at,
                                      enum afs_cb_promise_trace trace)
{
        atomic64_set(&vnode->cb_expires_at, expires_at);
        trace_afs_cb_promise(vnode, trace);
}

/*
 * Clear the callback promise on a vnode, returning true if it was promised.
 */
static inline bool afs_clear_cb_promise(struct afs_vnode *vnode,
                                        enum afs_cb_promise_trace trace)
{
        trace_afs_cb_promise(vnode, trace);
        return atomic64_xchg(&vnode->cb_expires_at, AFS_NO_CB_PROMISE) != AFS_NO_CB_PROMISE;
}

/*
 * Mark a directory as being invalid.
 */
static inline void afs_invalidate_dir(struct afs_vnode *dvnode,
                                      enum afs_dir_invalid_trace trace)
{
        if (test_and_clear_bit(AFS_VNODE_DIR_VALID, &dvnode->flags)) {
                trace_afs_dir_invalid(dvnode, trace);
                afs_stat_v(dvnode, n_inval);
        }
}

/*****************************************************************************/
/*
 * debug tracing
 */
extern unsigned afs_debug;

#define dbgprintk(FMT,...) \
        printk("[%-6.6s] "FMT"\n", current->comm ,##__VA_ARGS__)

#define kenter(FMT,...)        dbgprintk("==> %s("FMT")",__func__ ,##__VA_ARGS__)
#define kleave(FMT,...)        dbgprintk("<== %s()"FMT"",__func__ ,##__VA_ARGS__)
#define kdebug(FMT,...)        dbgprintk("    "FMT ,##__VA_ARGS__)


#if defined(__KDEBUG)
#define _enter(FMT,...)        kenter(FMT,##__VA_ARGS__)
#define _leave(FMT,...)        kleave(FMT,##__VA_ARGS__)
#define _debug(FMT,...)        kdebug(FMT,##__VA_ARGS__)

#elif defined(CONFIG_AFS_DEBUG)
#define AFS_DEBUG_KENTER        0x01
#define AFS_DEBUG_KLEAVE        0x02
#define AFS_DEBUG_KDEBUG        0x04

#define _enter(FMT,...)                                        \
do {                                                        \
        if (unlikely(afs_debug & AFS_DEBUG_KENTER))        \
                kenter(FMT,##__VA_ARGS__);                \
} while (0)

#define _leave(FMT,...)                                        \
do {                                                        \
        if (unlikely(afs_debug & AFS_DEBUG_KLEAVE))        \
                kleave(FMT,##__VA_ARGS__);                \
} while (0)

#define _debug(FMT,...)                                        \
do {                                                        \
        if (unlikely(afs_debug & AFS_DEBUG_KDEBUG))        \
                kdebug(FMT,##__VA_ARGS__);                \
} while (0)

#else
#define _enter(FMT,...)        no_printk("==> %s("FMT")",__func__ ,##__VA_ARGS__)
#define _leave(FMT,...)        no_printk("<== %s()"FMT"",__func__ ,##__VA_ARGS__)
#define _debug(FMT,...)        no_printk("    "FMT ,##__VA_ARGS__)
#endif

/*
 * debug assertion checking
 */
#if 1 // defined(__KDEBUGALL)

#define ASSERT(X)                                                \
do {                                                                \
        if (unlikely(!(X))) {                                        \
                printk(KERN_ERR "\n");                                \
                printk(KERN_ERR "AFS: Assertion failed\n");        \
                BUG();                                                \
        }                                                        \
} while(0)

#define ASSERTCMP(X, OP, Y)                                                \
do {                                                                        \
        if (unlikely(!((X) OP (Y)))) {                                        \
                printk(KERN_ERR "\n");                                        \
                printk(KERN_ERR "AFS: Assertion failed\n");                \
                printk(KERN_ERR "%lu " #OP " %lu is false\n",                \
                       (unsigned long)(X), (unsigned long)(Y));                \
                printk(KERN_ERR "0x%lx " #OP " 0x%lx is false\n",        \
                       (unsigned long)(X), (unsigned long)(Y));                \
                BUG();                                                        \
        }                                                                \
} while(0)

#define ASSERTRANGE(L, OP1, N, OP2, H)                                        \
do {                                                                        \
        if (unlikely(!((L) OP1 (N)) || !((N) OP2 (H)))) {                \
                printk(KERN_ERR "\n");                                        \
                printk(KERN_ERR "AFS: Assertion failed\n");                \
                printk(KERN_ERR "%lu "#OP1" %lu "#OP2" %lu is false\n",        \
                       (unsigned long)(L), (unsigned long)(N),                \
                       (unsigned long)(H));                                \
                printk(KERN_ERR "0x%lx "#OP1" 0x%lx "#OP2" 0x%lx is false\n", \
                       (unsigned long)(L), (unsigned long)(N),                \
                       (unsigned long)(H));                                \
                BUG();                                                        \
        }                                                                \
} while(0)

#define ASSERTIF(C, X)                                                \
do {                                                                \
        if (unlikely((C) && !(X))) {                                \
                printk(KERN_ERR "\n");                                \
                printk(KERN_ERR "AFS: Assertion failed\n");        \
                BUG();                                                \
        }                                                        \
} while(0)

#define ASSERTIFCMP(C, X, OP, Y)                                        \
do {                                                                        \
        if (unlikely((C) && !((X) OP (Y)))) {                                \
                printk(KERN_ERR "\n");                                        \
                printk(KERN_ERR "AFS: Assertion failed\n");                \
                printk(KERN_ERR "%lu " #OP " %lu is false\n",                \
                       (unsigned long)(X), (unsigned long)(Y));                \
                printk(KERN_ERR "0x%lx " #OP " 0x%lx is false\n",        \
                       (unsigned long)(X), (unsigned long)(Y));                \
                BUG();                                                        \
        }                                                                \
} while(0)

#else

#define ASSERT(X)                                \
do {                                                \
} while(0)

#define ASSERTCMP(X, OP, Y)                        \
do {                                                \
} while(0)

#define ASSERTRANGE(L, OP1, N, OP2, H)                \
do {                                                \
} while(0)

#define ASSERTIF(C, X)                                \
do {                                                \
} while(0)

#define ASSERTIFCMP(C, X, OP, Y)                \
do {                                                \
} while(0)

#endif /* __KDEBUGALL */


















































































































































   46 








    3 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
// SPDX-License-Identifier: GPL-2.0
/*
 * Author: Andrei Vagin <avagin@openvz.org>
 * Author: Dmitry Safonov <dima@arista.com>
 */

#include <linux/cleanup.h>
#include <linux/mm.h>
#include <linux/time_namespace.h>
#include <linux/time.h>
#include <linux/vdso_datastore.h>

#include <vdso/clocksource.h>
#include <vdso/datapage.h>

#include "namespace_internal.h"

static struct timens_offset offset_from_ts(struct timespec64 off)
{
        struct timens_offset ret;

        ret.sec = off.tv_sec;
        ret.nsec = off.tv_nsec;

        return ret;
}

/*
 * A time namespace VVAR page has the same layout as the VVAR page which
 * contains the system wide VDSO data.
 *
 * For a normal task the VVAR pages are installed in the normal ordering:
 *     VVAR
 *     PVCLOCK
 *     HVCLOCK
 *     TIMENS   <- Not really required
 *
 * Now for a timens task the pages are installed in the following order:
 *     TIMENS
 *     PVCLOCK
 *     HVCLOCK
 *     VVAR
 *
 * The check for vdso_clock->clock_mode is in the unlikely path of
 * the seq begin magic. So for the non-timens case most of the time
 * 'seq' is even, so the branch is not taken.
 *
 * If 'seq' is odd, i.e. a concurrent update is in progress, the extra check
 * for vdso_clock->clock_mode is a non-issue. The task is spin waiting for the
 * update to finish and for 'seq' to become even anyway.
 *
 * Timens page has vdso_clock->clock_mode set to VDSO_CLOCKMODE_TIMENS which
 * enforces the time namespace handling path.
 */
static void timens_setup_vdso_clock_data(struct vdso_clock *vc,
                                         struct time_namespace *ns)
{
        struct timens_offset *offset = vc->offset;
        struct timens_offset monotonic = offset_from_ts(ns->offsets.monotonic);
        struct timens_offset boottime = offset_from_ts(ns->offsets.boottime);

        vc->seq                                = 1;
        vc->clock_mode                        = VDSO_CLOCKMODE_TIMENS;
        offset[CLOCK_MONOTONIC]                = monotonic;
        offset[CLOCK_MONOTONIC_RAW]        = monotonic;
        offset[CLOCK_MONOTONIC_COARSE]        = monotonic;
        offset[CLOCK_BOOTTIME]                = boottime;
        offset[CLOCK_BOOTTIME_ALARM]        = boottime;
}

struct page *find_timens_vvar_page(struct vm_area_struct *vma)
{
        if (likely(vma->vm_mm == current->mm))
                return current->nsproxy->time_ns->vvar_page;

        /*
         * vvar_fault() protects this from being called through remote interfaces like
         * /proc/$pid/mem or process_vm_{readv,writev}().
         */

        WARN(1, "vvar_page accessed remotely");

        return NULL;
}

static void timens_set_vvar_page(struct task_struct *task,
                                struct time_namespace *ns)
{
        struct vdso_time_data *vdata;
        struct vdso_clock *vc;
        unsigned int i;

        if (ns == &init_time_ns)
                return;

        /* Fast-path, taken by every task in namespace except the first. */
        if (likely(ns->frozen_offsets))
                return;

        guard(mutex)(&timens_offset_lock);
        /* Nothing to-do: vvar_page has been already initialized. */
        if (ns->frozen_offsets)
                return;

        ns->frozen_offsets = true;
        vdata = page_address(ns->vvar_page);
        vc = vdata->clock_data;

        for (i = 0; i < CS_BASES; i++)
                timens_setup_vdso_clock_data(&vc[i], ns);

        if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) {
                for (i = 0; i < ARRAY_SIZE(vdata->aux_clock_data); i++)
                        timens_setup_vdso_clock_data(&vdata->aux_clock_data[i], ns);
        }
}

/*
 * The vvar page layout depends on whether a task belongs to the root or
 * non-root time namespace. Whenever a task changes its namespace, the VVAR
 * page tables are cleared and then they will be re-faulted with a
 * corresponding layout.
 * See also the comment near timens_setup_vdso_clock_data() for details.
 */
static int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
{
        struct mm_struct *mm = task->mm;
        struct vm_area_struct *vma;
        VMA_ITERATOR(vmi, mm, 0);

        guard(mmap_read_lock)(mm);
        for_each_vma(vmi, vma) {
                if (vma_is_special_mapping(vma, &vdso_vvar_mapping))
                        zap_vma(vma);
        }
        return 0;
}

void timens_commit(struct task_struct *tsk, struct time_namespace *ns)
{
        timens_set_vvar_page(tsk, ns);
        vdso_join_timens(tsk, ns);
}

int timens_vdso_alloc_vvar_page(struct time_namespace *ns)
{
        ns->vvar_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
        if (!ns->vvar_page)
                return -ENOMEM;

        return 0;
}

void timens_vdso_free_vvar_page(struct time_namespace *ns)
{
        __free_page(ns->vvar_page);
}

















































































































































































































  147 









































   73 



   79 





































  149 

   63 


   64 
   32 






  149 

   63 


   64 
   32 






   46 

   18 


   12 
   19 



































































   20 







   18 
   12 
   19 







   56 



   56 






    2 








  150 






















   66 

















   86 













    7 













   10 













   10 



   10 

   78 








   64 





















   82 



   63 
   63 






   86 



    6 
    6 








   86 



   86 
   86 
   86 
   86 
































   93 





   71 







    8 







   32 











    7 

   94 
   86 










   94 















   94 











   70 



   70 




   70 

































   86 






   86 











   86 
















   86 



































   94 



















































   72 







   70 







   29 


    1 












    7 



    7 















    7 






    7 








   24 






   14 





    1 




















































   17 



    8 

    7 


    4 
















  152 
































   87 












   36 







   36 
   36 






  150 
























  150 






   67 








   67 
   60 





   67 











   70 





   87 





   36 
   36 
   36 





   36 
   36 
   36 













    7 
   67 




























































   43 


   43 


   43 











  126 
  126 
    6 





  123 


  126 



  126 








  126 







  126 





  126 
  126 


  126 





   76 



  126 



    4 
















  126 



















































   35 





   35 



   35 

   30 
    6 




   35 









   35 





































   35 

















   35 















   35 
















   35 




















































   30 

   30 



































































   21 











   21 











   21 

   21 






   21 






   18 


   60 
   60 






   47 




   47 






   47 


   26 
   25 












   17 






   35 






   47 

   47 
   47 
   47 
   35 
   17 







   30 




   34 









   79 






   79 


   79 




































   79 























   79 











   79 



   53 
   30 




   79 


   79 








































































    7 



    7 
    7 




    7 
    7 
    6 












































































































































   88 






   27 













   79 















   79 










































   61 






































































   47 






















   47 





   47 


   23 








   30 






   31 

   11 



   30 












   11 





















   20 
    4 











   31 




















   21 




















   20 



    1 


    6 
















































  132 





  132 
   54 


   90 


   90 




   90 




   82 











   17 

















   82 
















   30 





















   16 





   17 
























































































































































































































































































































  129 




  129 


  129 




  129 











  129 
  126 

  126 

   61 
   53 




   53 
   53 


  105 
   28 





   28 
  105 













  126 












   16 


   16 
































  129 






  129 


































  129 
  108 































  129 








  129 





















  126 


















   16 






   16 















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   37 

   37 
   37 
   37 





















































    5 





    5 


    5 


    5 
    5 



















   93 


   64 



























   47 


   37 







   21 







   21 
   21 










   12 
   11 
   11 








   15 





    6 










   46 








   46 
   30 


   19 
   19 

   19 

    4 








   19 



   29 






   21 








   21 
   21 





   21 
































































  126 
  126 


   92 
   38 












































































































































































































































































































































































   46 







   46 



   46 
   46 








   46 

   46 

   46 
   46 

    5 



   41 








   18 






























   40 














   40 
   40 
   10 
   37 

   40 




   40 
   40 














   28 
   17 

























   17 


   40 
   40 



   40 









   80 
   42 


   46 


   46 




   80 


   42 












   46 





   46 





















































































   35 






























   35 













   35 
   35 





   35 

   35 



























































































































































   59 




   59 
















   59 



























   16 



   16 








  144 




























  144 

  144 
   16 





























   61 

   72 


















   16 











   59 















   71 

































  102 





   31 


  126 






   71 

   59 





  126 

  126 

  126 
  102 




   31 















  126 


  126 







  129 



















   10 























  147 











  147 
























  147 












  144 


















































   30 








































   20 
   19 





   10 


   30 






   30 














   20 
   19 






   10 






   10 



   10 































   30 




















































































































































































































































































































































































































































  150 


















  150 






  150 








   12 







   12 








   36 












   47 



   12 



   36 







   47 










   14 













   36 


































   10 






   14 









   14 









   14 


   14 





    2 
   12 







   66 

















   66 




   66 


















































   89 





























   46 



































































   41 
   41 

   41 
   41 




   41 














































   46 





   46 

   46 









































  147 





  147 
  147 


  147 





  147 


  147 



  147 


  147 
















   46 





   46 
   46 

   46 
    5 










   41 













   46 











































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127
7128
7129
7130
7131
7132
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
7219
7220
7221
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231
7232
7233
7234
7235
7236
7237
7238
7239
7240
7241
7242
7243
7244
7245
7246
7247
7248
7249
7250
7251
7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317
7318
7319
7320
7321
7322
7323
7324
7325
7326
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342
7343
7344
7345
7346
7347
7348
7349
7350
7351
7352
7353
7354
7355
7356
7357
7358
7359
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385
7386
7387
7388
7389
7390
7391
7392
7393
7394
7395
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412
7413
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Generic hugetlb support.
 * (C) Nadia Yvette Chambers, April 2004
 */
#include <linux/list.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/seq_file.h>
#include <linux/highmem.h>
#include <linux/mmu_notifier.h>
#include <linux/nodemask.h>
#include <linux/pagemap.h>
#include <linux/mempolicy.h>
#include <linux/compiler.h>
#include <linux/cpumask.h>
#include <linux/cpuset.h>
#include <linux/mutex.h>
#include <linux/memblock.h>
#include <linux/minmax.h>
#include <linux/slab.h>
#include <linux/sched/mm.h>
#include <linux/mmdebug.h>
#include <linux/sched/signal.h>
#include <linux/rmap.h>
#include <linux/string_choices.h>
#include <linux/string_helpers.h>
#include <linux/swap.h>
#include <linux/leafops.h>
#include <linux/jhash.h>
#include <linux/numa.h>
#include <linux/llist.h>
#include <linux/cma.h>
#include <linux/migrate.h>
#include <linux/nospec.h>
#include <linux/delayacct.h>
#include <linux/memory.h>
#include <linux/mm_inline.h>
#include <linux/padata.h>
#include <linux/pgalloc.h>

#include <asm/page.h>
#include <asm/tlb.h>
#include <asm/setup.h>

#include <linux/io.h>
#include <linux/node.h>
#include <linux/page_owner.h>
#include "internal.h"
#include "page_alloc.h"
#include "hugetlb_vmemmap.h"
#include "hugetlb_cma.h"
#include "hugetlb_internal.h"
#include "mm_init.h"
#include <linux/page-isolation.h>

int hugetlb_max_hstate __read_mostly;
unsigned int default_hstate_idx;
struct hstate hstates[HUGE_MAX_HSTATE];

__initdata nodemask_t hugetlb_bootmem_nodes;
__initdata struct list_head huge_boot_pages[MAX_NUMNODES];

/*
 * Due to ordering constraints across the init code for various
 * architectures, hugetlb hstate cmdline parameters can't simply
 * be early_param. early_param might call the setup function
 * before valid hugetlb page sizes are determined, leading to
 * incorrect rejection of valid hugepagesz= options.
 *
 * So, record the parameters early and consume them whenever the
 * init code is ready for them, by calling hugetlb_parse_params().
 */

/* one (hugepagesz=,hugepages=) pair per hstate, one default_hugepagesz */
#define HUGE_MAX_CMDLINE_ARGS        (2 * HUGE_MAX_HSTATE + 1)
struct hugetlb_cmdline {
        char *val;
        int (*setup)(char *val);
};

/* for command line parsing */
static struct hstate * __initdata parsed_hstate;
static unsigned long __initdata default_hstate_max_huge_pages;
static bool __initdata parsed_valid_hugepagesz = true;
static bool __initdata parsed_default_hugepagesz;
static unsigned int default_hugepages_in_node[MAX_NUMNODES] __initdata;
static unsigned long hugepage_allocation_threads __initdata;

static char hstate_cmdline_buf[COMMAND_LINE_SIZE] __initdata;
static int hstate_cmdline_index __initdata;
static struct hugetlb_cmdline hugetlb_params[HUGE_MAX_CMDLINE_ARGS] __initdata;
static int hugetlb_param_index __initdata;
static __init int hugetlb_add_param(char *s, int (*setup)(char *val));
static __init void hugetlb_parse_params(void);

#define hugetlb_early_param(str, func) \
static __init int func##args(char *s) \
{ \
        return hugetlb_add_param(s, func); \
} \
early_param(str, func##args)

/*
 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
 * free_huge_pages, and surplus_huge_pages.
 */
__cacheline_aligned_in_smp DEFINE_SPINLOCK(hugetlb_lock);

/*
 * Serializes faults on the same logical page.  This is used to
 * prevent spurious OOMs when the hugepage pool is fully utilized.
 */
static int num_fault_mutexes __ro_after_init;
struct mutex *hugetlb_fault_mutex_table __ro_after_init;

/* Forward declaration */
static int hugetlb_acct_memory(struct hstate *h, long delta);
static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
static int __huge_pmd_unshare(struct mmu_gather *tlb,
                struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
                bool check_locks);
static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
                unsigned long start, unsigned long end, bool take_locks);
static struct resv_map *vma_resv_map(struct vm_area_struct *vma);

static inline bool subpool_is_free(struct hugepage_subpool *spool)
{
        if (spool->count)
                return false;
        if (spool->max_hpages != -1)
                return spool->used_hpages == 0;
        if (spool->min_hpages != -1)
                return spool->rsv_hpages == spool->min_hpages;

        return true;
}

static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
                                                unsigned long irq_flags)
{
        bool free_subpool = subpool_is_free(spool);

        /* If no pages are used, and no other handles to the subpool
         * remain, give up any reservations based on minimum size and
         * free the subpool */
        spin_unlock_irqrestore(&spool->lock, irq_flags);

        if (free_subpool) {
                if (spool->min_hpages != -1)
                        hugetlb_acct_memory(spool->hstate,
                                                -spool->min_hpages);
                kfree(spool);
        }
}

struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
                                                long min_hpages)
{
        struct hugepage_subpool *spool;

        spool = kzalloc_obj(*spool);
        if (!spool)
                return NULL;

        spin_lock_init(&spool->lock);
        spool->count = 1;
        spool->max_hpages = max_hpages;
        spool->hstate = h;
        spool->min_hpages = min_hpages;

        if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
                kfree(spool);
                return NULL;
        }
        spool->rsv_hpages = min_hpages;

        return spool;
}

void hugepage_put_subpool(struct hugepage_subpool *spool)
{
        unsigned long flags;

        if (!spool)
                return;

        spin_lock_irqsave(&spool->lock, flags);
        BUG_ON(!spool->count);
        spool->count--;
        unlock_or_release_subpool(spool, flags);
}

/*
 * Subpool accounting for allocating and reserving pages.
 * Return -ENOMEM if there are not enough resources to satisfy the
 * request.  Otherwise, return the number of pages by which the
 * global pools must be adjusted (upward).  The returned value may
 * only be different than the passed value (delta) in the case where
 * a subpool minimum size must be maintained.
 */
static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
                                      long delta)
{
        long ret = delta;

        if (!spool)
                return ret;

        spin_lock_irq(&spool->lock);

        if (spool->max_hpages != -1) {                /* maximum size accounting */
                if ((spool->used_hpages + delta) <= spool->max_hpages)
                        spool->used_hpages += delta;
                else {
                        ret = -ENOMEM;
                        goto unlock_ret;
                }
        }

        /* minimum size accounting */
        if (spool->min_hpages != -1 && spool->rsv_hpages) {
                if (delta > spool->rsv_hpages) {
                        /*
                         * Asking for more reserves than those already taken on
                         * behalf of subpool.  Return difference.
                         */
                        ret = delta - spool->rsv_hpages;
                        spool->rsv_hpages = 0;
                } else {
                        ret = 0;        /* reserves already accounted for */
                        spool->rsv_hpages -= delta;
                }
        }

unlock_ret:
        spin_unlock_irq(&spool->lock);
        return ret;
}

/*
 * Subpool accounting for freeing and unreserving pages.
 * Return the number of global page reservations that must be dropped.
 * The return value may only be different than the passed value (delta)
 * in the case where a subpool minimum size must be maintained.
 */
static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
                                       long delta)
{
        long ret = delta;
        unsigned long flags;

        if (!spool)
                return delta;

        spin_lock_irqsave(&spool->lock, flags);

        if (spool->max_hpages != -1)                /* maximum size accounting */
                spool->used_hpages -= delta;

         /* minimum size accounting */
        if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
                if (spool->rsv_hpages + delta <= spool->min_hpages)
                        ret = 0;
                else
                        ret = spool->rsv_hpages + delta - spool->min_hpages;

                spool->rsv_hpages += delta;
                if (spool->rsv_hpages > spool->min_hpages)
                        spool->rsv_hpages = spool->min_hpages;
        }

        /*
         * If hugetlbfs_put_super couldn't free spool due to an outstanding
         * quota reference, free it now.
         */
        unlock_or_release_subpool(spool, flags);

        return ret;
}

static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
{
        return subpool_inode(file_inode(vma->vm_file));
}

/*
 * hugetlb vma_lock helper routines
 */
void hugetlb_vma_lock_read(struct vm_area_struct *vma)
{
        if (__vma_shareable_lock(vma)) {
                struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

                down_read(&vma_lock->rw_sema);
        } else if (__vma_private_lock(vma)) {
                struct resv_map *resv_map = vma_resv_map(vma);

                down_read(&resv_map->rw_sema);
        }
}

void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
{
        if (__vma_shareable_lock(vma)) {
                struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

                up_read(&vma_lock->rw_sema);
        } else if (__vma_private_lock(vma)) {
                struct resv_map *resv_map = vma_resv_map(vma);

                up_read(&resv_map->rw_sema);
        }
}

void hugetlb_vma_lock_write(struct vm_area_struct *vma)
{
        if (__vma_shareable_lock(vma)) {
                struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

                down_write(&vma_lock->rw_sema);
        } else if (__vma_private_lock(vma)) {
                struct resv_map *resv_map = vma_resv_map(vma);

                down_write(&resv_map->rw_sema);
        }
}

void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
{
        if (__vma_shareable_lock(vma)) {
                struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

                up_write(&vma_lock->rw_sema);
        } else if (__vma_private_lock(vma)) {
                struct resv_map *resv_map = vma_resv_map(vma);

                up_write(&resv_map->rw_sema);
        }
}

int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
{

        if (__vma_shareable_lock(vma)) {
                struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

                return down_write_trylock(&vma_lock->rw_sema);
        } else if (__vma_private_lock(vma)) {
                struct resv_map *resv_map = vma_resv_map(vma);

                return down_write_trylock(&resv_map->rw_sema);
        }

        return 1;
}

void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
{
        if (__vma_shareable_lock(vma)) {
                struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

                lockdep_assert_held(&vma_lock->rw_sema);
        } else if (__vma_private_lock(vma)) {
                struct resv_map *resv_map = vma_resv_map(vma);

                lockdep_assert_held(&resv_map->rw_sema);
        }
}

void hugetlb_vma_lock_release(struct kref *kref)
{
        struct hugetlb_vma_lock *vma_lock = container_of(kref,
                        struct hugetlb_vma_lock, refs);

        kfree(vma_lock);
}

static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock)
{
        struct vm_area_struct *vma = vma_lock->vma;

        /*
         * vma_lock structure may or not be released as a result of put,
         * it certainly will no longer be attached to vma so clear pointer.
         * Semaphore synchronizes access to vma_lock->vma field.
         */
        vma_lock->vma = NULL;
        vma->vm_private_data = NULL;
        up_write(&vma_lock->rw_sema);
        kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
}

static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma)
{
        if (__vma_shareable_lock(vma)) {
                struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

                __hugetlb_vma_unlock_write_put(vma_lock);
        } else if (__vma_private_lock(vma)) {
                struct resv_map *resv_map = vma_resv_map(vma);

                /* no free for anon vmas, but still need to unlock */
                up_write(&resv_map->rw_sema);
        }
}

static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
{
        /*
         * Only present in sharable vmas.
         */
        if (!vma || !__vma_shareable_lock(vma))
                return;

        if (vma->vm_private_data) {
                struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

                down_write(&vma_lock->rw_sema);
                __hugetlb_vma_unlock_write_put(vma_lock);
        }
}

static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
{
        struct hugetlb_vma_lock *vma_lock;

        /* Only establish in (flags) sharable vmas */
        if (!vma || !(vma->vm_flags & VM_MAYSHARE))
                return;

        /* Should never get here with non-NULL vm_private_data */
        if (vma->vm_private_data)
                return;

        vma_lock = kmalloc_obj(*vma_lock);
        if (!vma_lock) {
                /*
                 * If we can not allocate structure, then vma can not
                 * participate in pmd sharing.  This is only a possible
                 * performance enhancement and memory saving issue.
                 * However, the lock is also used to synchronize page
                 * faults with truncation.  If the lock is not present,
                 * unlikely races could leave pages in a file past i_size
                 * until the file is removed.  Warn in the unlikely case of
                 * allocation failure.
                 */
                pr_warn_once("HugeTLB: unable to allocate vma specific lock\n");
                return;
        }

        kref_init(&vma_lock->refs);
        init_rwsem(&vma_lock->rw_sema);
        vma_lock->vma = vma;
        vma->vm_private_data = vma_lock;
}

/* Helper that removes a struct file_region from the resv_map cache and returns
 * it for use.
 */
static struct file_region *
get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
{
        struct file_region *nrg;

        VM_BUG_ON(resv->region_cache_count <= 0);

        resv->region_cache_count--;
        nrg = list_first_entry(&resv->region_cache, struct file_region, link);
        list_del(&nrg->link);

        nrg->from = from;
        nrg->to = to;

        return nrg;
}

static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg,
                                              struct file_region *rg)
{
#ifdef CONFIG_CGROUP_HUGETLB
        nrg->reservation_counter = rg->reservation_counter;
        nrg->css = rg->css;
        if (rg->css)
                css_get(rg->css);
#endif
}

/* Helper that records hugetlb_cgroup uncharge info. */
static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg,
                                                struct hstate *h,
                                                struct resv_map *resv,
                                                struct file_region *nrg)
{
#ifdef CONFIG_CGROUP_HUGETLB
        if (h_cg) {
                nrg->reservation_counter =
                        &h_cg->rsvd_hugepage[hstate_index(h)];
                nrg->css = &h_cg->css;
                /*
                 * The caller will hold exactly one h_cg->css reference for the
                 * whole contiguous reservation region. But this area might be
                 * scattered when there are already some file_regions reside in
                 * it. As a result, many file_regions may share only one css
                 * reference. In order to ensure that one file_region must hold
                 * exactly one h_cg->css reference, we should do css_get for
                 * each file_region and leave the reference held by caller
                 * untouched.
                 */
                css_get(&h_cg->css);
                if (!resv->pages_per_hpage)
                        resv->pages_per_hpage = pages_per_huge_page(h);
                /* pages_per_hpage should be the same for all entries in
                 * a resv_map.
                 */
                VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));
        } else {
                nrg->reservation_counter = NULL;
                nrg->css = NULL;
        }
#endif
}

static void put_uncharge_info(struct file_region *rg)
{
#ifdef CONFIG_CGROUP_HUGETLB
        if (rg->css)
                css_put(rg->css);
#endif
}

static bool has_same_uncharge_info(struct file_region *rg,
                                   struct file_region *org)
{
#ifdef CONFIG_CGROUP_HUGETLB
        return rg->reservation_counter == org->reservation_counter &&
               rg->css == org->css;

#else
        return true;
#endif
}

static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
{
        struct file_region *nrg, *prg;

        prg = list_prev_entry(rg, link);
        if (&prg->link != &resv->regions && prg->to == rg->from &&
            has_same_uncharge_info(prg, rg)) {
                prg->to = rg->to;

                list_del(&rg->link);
                put_uncharge_info(rg);
                kfree(rg);

                rg = prg;
        }

        nrg = list_next_entry(rg, link);
        if (&nrg->link != &resv->regions && nrg->from == rg->to &&
            has_same_uncharge_info(nrg, rg)) {
                nrg->from = rg->from;

                list_del(&rg->link);
                put_uncharge_info(rg);
                kfree(rg);
        }
}

static inline long
hugetlb_resv_map_add(struct resv_map *map, struct list_head *rg, long from,
                     long to, struct hstate *h, struct hugetlb_cgroup *cg,
                     long *regions_needed)
{
        struct file_region *nrg;

        if (!regions_needed) {
                nrg = get_file_region_entry_from_cache(map, from, to);
                record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg);
                list_add(&nrg->link, rg);
                coalesce_file_region(map, nrg);
        } else {
                *regions_needed += 1;
        }

        return to - from;
}

/*
 * Must be called with resv->lock held.
 *
 * Calling this with regions_needed != NULL will count the number of pages
 * to be added but will not modify the linked list. And regions_needed will
 * indicate the number of file_regions needed in the cache to carry out to add
 * the regions for this range.
 */
static long add_reservation_in_range(struct resv_map *resv, long f, long t,
                                     struct hugetlb_cgroup *h_cg,
                                     struct hstate *h, long *regions_needed)
{
        long add = 0;
        struct list_head *head = &resv->regions;
        long last_accounted_offset = f;
        struct file_region *iter, *trg = NULL;
        struct list_head *rg = NULL;

        if (regions_needed)
                *regions_needed = 0;

        /* In this loop, we essentially handle an entry for the range
         * [last_accounted_offset, iter->from), at every iteration, with some
         * bounds checking.
         */
        list_for_each_entry_safe(iter, trg, head, link) {
                /* Skip irrelevant regions that start before our range. */
                if (iter->from < f) {
                        /* If this region ends after the last accounted offset,
                         * then we need to update last_accounted_offset.
                         */
                        if (iter->to > last_accounted_offset)
                                last_accounted_offset = iter->to;
                        continue;
                }

                /* When we find a region that starts beyond our range, we've
                 * finished.
                 */
                if (iter->from >= t) {
                        rg = iter->link.prev;
                        break;
                }

                /* Add an entry for last_accounted_offset -> iter->from, and
                 * update last_accounted_offset.
                 */
                if (iter->from > last_accounted_offset)
                        add += hugetlb_resv_map_add(resv, iter->link.prev,
                                                    last_accounted_offset,
                                                    iter->from, h, h_cg,
                                                    regions_needed);

                last_accounted_offset = iter->to;
        }

        /* Handle the case where our range extends beyond
         * last_accounted_offset.
         */
        if (!rg)
                rg = head->prev;
        if (last_accounted_offset < t)
                add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,
                                            t, h, h_cg, regions_needed);

        return add;
}

/* Must be called with resv->lock acquired. Will drop lock to allocate entries.
 */
static int allocate_file_region_entries(struct resv_map *resv,
                                        int regions_needed)
        __must_hold(&resv->lock)
{
        LIST_HEAD(allocated_regions);
        int to_allocate = 0, i = 0;
        struct file_region *trg = NULL, *rg = NULL;

        VM_BUG_ON(regions_needed < 0);

        /*
         * Check for sufficient descriptors in the cache to accommodate
         * the number of in progress add operations plus regions_needed.
         *
         * This is a while loop because when we drop the lock, some other call
         * to region_add or region_del may have consumed some region_entries,
         * so we keep looping here until we finally have enough entries for
         * (adds_in_progress + regions_needed).
         */
        while (resv->region_cache_count <
               (resv->adds_in_progress + regions_needed)) {
                to_allocate = resv->adds_in_progress + regions_needed -
                              resv->region_cache_count;

                /* At this point, we should have enough entries in the cache
                 * for all the existing adds_in_progress. We should only be
                 * needing to allocate for regions_needed.
                 */
                VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);

                spin_unlock(&resv->lock);
                for (i = 0; i < to_allocate; i++) {
                        trg = kmalloc_obj(*trg);
                        if (!trg)
                                goto out_of_memory;
                        list_add(&trg->link, &allocated_regions);
                }

                spin_lock(&resv->lock);

                list_splice_init(&allocated_regions, &resv->region_cache);
                resv->region_cache_count += to_allocate;
        }

        return 0;

out_of_memory:
        list_for_each_entry_safe(rg, trg, &allocated_regions, link) {
                list_del(&rg->link);
                kfree(rg);
        }
        return -ENOMEM;
}

/*
 * Add the huge page range represented by [f, t) to the reserve
 * map.  Regions will be taken from the cache to fill in this range.
 * Sufficient regions should exist in the cache due to the previous
 * call to region_chg with the same range, but in some cases the cache will not
 * have sufficient entries due to races with other code doing region_add or
 * region_del.  The extra needed entries will be allocated.
 *
 * regions_needed is the out value provided by a previous call to region_chg.
 *
 * Return the number of new huge pages added to the map.  This number is greater
 * than or equal to zero.  If file_region entries needed to be allocated for
 * this operation and we were not able to allocate, it returns -ENOMEM.
 * region_add of regions of length 1 never allocate file_regions and cannot
 * fail; region_chg will always allocate at least 1 entry and a region_add for
 * 1 page will only require at most 1 entry.
 */
static long region_add(struct resv_map *resv, long f, long t,
                       long in_regions_needed, struct hstate *h,
                       struct hugetlb_cgroup *h_cg)
{
        long add = 0, actual_regions_needed = 0;

        spin_lock(&resv->lock);
retry:

        /* Count how many regions are actually needed to execute this add. */
        add_reservation_in_range(resv, f, t, NULL, NULL,
                                 &actual_regions_needed);

        /*
         * Check for sufficient descriptors in the cache to accommodate
         * this add operation. Note that actual_regions_needed may be greater
         * than in_regions_needed, as the resv_map may have been modified since
         * the region_chg call. In this case, we need to make sure that we
         * allocate extra entries, such that we have enough for all the
         * existing adds_in_progress, plus the excess needed for this
         * operation.
         */
        if (actual_regions_needed > in_regions_needed &&
            resv->region_cache_count <
                    resv->adds_in_progress +
                            (actual_regions_needed - in_regions_needed)) {
                /* region_add operation of range 1 should never need to
                 * allocate file_region entries.
                 */
                VM_BUG_ON(t - f <= 1);

                if (allocate_file_region_entries(
                            resv, actual_regions_needed - in_regions_needed)) {
                        return -ENOMEM;
                }

                goto retry;
        }

        add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);

        resv->adds_in_progress -= in_regions_needed;

        spin_unlock(&resv->lock);
        return add;
}

/*
 * Examine the existing reserve map and determine how many
 * huge pages in the specified range [f, t) are NOT currently
 * represented.  This routine is called before a subsequent
 * call to region_add that will actually modify the reserve
 * map to add the specified range [f, t).  region_chg does
 * not change the number of huge pages represented by the
 * map.  A number of new file_region structures is added to the cache as a
 * placeholder, for the subsequent region_add call to use. At least 1
 * file_region structure is added.
 *
 * out_regions_needed is the number of regions added to the
 * resv->adds_in_progress.  This value needs to be provided to a follow up call
 * to region_add or region_abort for proper accounting.
 *
 * Returns the number of huge pages that need to be added to the existing
 * reservation map for the range [f, t).  This number is greater or equal to
 * zero.  -ENOMEM is returned if a new file_region structure or cache entry
 * is needed and can not be allocated.
 */
static long region_chg(struct resv_map *resv, long f, long t,
                       long *out_regions_needed)
{
        long chg = 0;

        spin_lock(&resv->lock);

        /* Count how many hugepages in this range are NOT represented. */
        chg = add_reservation_in_range(resv, f, t, NULL, NULL,
                                       out_regions_needed);

        if (*out_regions_needed == 0)
                *out_regions_needed = 1;

        if (allocate_file_region_entries(resv, *out_regions_needed))
                return -ENOMEM;

        resv->adds_in_progress += *out_regions_needed;

        spin_unlock(&resv->lock);
        return chg;
}

/*
 * Abort the in progress add operation.  The adds_in_progress field
 * of the resv_map keeps track of the operations in progress between
 * calls to region_chg and region_add.  Operations are sometimes
 * aborted after the call to region_chg.  In such cases, region_abort
 * is called to decrement the adds_in_progress counter. regions_needed
 * is the value returned by the region_chg call, it is used to decrement
 * the adds_in_progress counter.
 *
 * NOTE: The range arguments [f, t) are not needed or used in this
 * routine.  They are kept to make reading the calling code easier as
 * arguments will match the associated region_chg call.
 */
static void region_abort(struct resv_map *resv, long f, long t,
                         long regions_needed)
{
        spin_lock(&resv->lock);
        VM_BUG_ON(!resv->region_cache_count);
        resv->adds_in_progress -= regions_needed;
        spin_unlock(&resv->lock);
}

/*
 * Delete the specified range [f, t) from the reserve map.  If the
 * t parameter is LONG_MAX, this indicates that ALL regions after f
 * should be deleted.  Locate the regions which intersect [f, t)
 * and either trim, delete or split the existing regions.
 *
 * Returns the number of huge pages deleted from the reserve map.
 * In the normal case, the return value is zero or more.  In the
 * case where a region must be split, a new region descriptor must
 * be allocated.  If the allocation fails, -ENOMEM will be returned.
 * NOTE: If the parameter t == LONG_MAX, then we will never split
 * a region and possibly return -ENOMEM.  Callers specifying
 * t == LONG_MAX do not need to check for -ENOMEM error.
 */
static long region_del(struct resv_map *resv, long f, long t)
{
        struct list_head *head = &resv->regions;
        struct file_region *rg, *trg;
        struct file_region *nrg = NULL;
        long del = 0;

retry:
        spin_lock(&resv->lock);
        list_for_each_entry_safe(rg, trg, head, link) {
                /*
                 * Skip regions before the range to be deleted.  file_region
                 * ranges are normally of the form [from, to).  However, there
                 * may be a "placeholder" entry in the map which is of the form
                 * (from, to) with from == to.  Check for placeholder entries
                 * at the beginning of the range to be deleted.
                 */
                if (rg->to <= f && (rg->to != rg->from || rg->to != f))
                        continue;

                if (rg->from >= t)
                        break;

                if (f > rg->from && t < rg->to) { /* Must split region */
                        /*
                         * Check for an entry in the cache before dropping
                         * lock and attempting allocation.
                         */
                        if (!nrg &&
                            resv->region_cache_count > resv->adds_in_progress) {
                                nrg = list_first_entry(&resv->region_cache,
                                                        struct file_region,
                                                        link);
                                list_del(&nrg->link);
                                resv->region_cache_count--;
                        }

                        if (!nrg) {
                                spin_unlock(&resv->lock);
                                nrg = kmalloc_obj(*nrg);
                                if (!nrg)
                                        return -ENOMEM;
                                goto retry;
                        }

                        del += t - f;
                        hugetlb_cgroup_uncharge_file_region(
                                resv, rg, t - f, false);

                        /* New entry for end of split region */
                        nrg->from = t;
                        nrg->to = rg->to;

                        copy_hugetlb_cgroup_uncharge_info(nrg, rg);

                        INIT_LIST_HEAD(&nrg->link);

                        /* Original entry is trimmed */
                        rg->to = f;

                        list_add(&nrg->link, &rg->link);
                        nrg = NULL;
                        break;
                }

                if (f <= rg->from && t >= rg->to) { /* Remove entire region */
                        del += rg->to - rg->from;
                        hugetlb_cgroup_uncharge_file_region(resv, rg,
                                                            rg->to - rg->from, true);
                        list_del(&rg->link);
                        kfree(rg);
                        continue;
                }

                if (f <= rg->from) {        /* Trim beginning of region */
                        hugetlb_cgroup_uncharge_file_region(resv, rg,
                                                            t - rg->from, false);

                        del += t - rg->from;
                        rg->from = t;
                } else {                /* Trim end of region */
                        hugetlb_cgroup_uncharge_file_region(resv, rg,
                                                            rg->to - f, false);

                        del += rg->to - f;
                        rg->to = f;
                }
        }

        spin_unlock(&resv->lock);
        kfree(nrg);
        return del;
}

/*
 * A rare out of memory error was encountered which prevented removal of
 * the reserve map region for a page.  The huge page itself was free'ed
 * and removed from the page cache.  This routine will adjust the subpool
 * usage count, and the global reserve count if needed.  By incrementing
 * these counts, the reserve map entry which could not be deleted will
 * appear as a "reserved" entry instead of simply dangling with incorrect
 * counts.
 */
void hugetlb_fix_reserve_counts(struct inode *inode)
{
        struct hugepage_subpool *spool = subpool_inode(inode);
        long rsv_adjust;
        bool reserved = false;

        rsv_adjust = hugepage_subpool_get_pages(spool, 1);
        if (rsv_adjust > 0) {
                struct hstate *h = hstate_inode(inode);

                if (!hugetlb_acct_memory(h, 1))
                        reserved = true;
        } else if (!rsv_adjust) {
                reserved = true;
        }

        if (!reserved)
                pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
}

/*
 * Count and return the number of huge pages in the reserve map
 * that intersect with the range [f, t).
 */
static long region_count(struct resv_map *resv, long f, long t)
{
        struct list_head *head = &resv->regions;
        struct file_region *rg;
        long chg = 0;

        spin_lock(&resv->lock);
        /* Locate each segment we overlap with, and count that overlap. */
        list_for_each_entry(rg, head, link) {
                long seg_from;
                long seg_to;

                if (rg->to <= f)
                        continue;
                if (rg->from >= t)
                        break;

                seg_from = max(rg->from, f);
                seg_to = min(rg->to, t);

                chg += seg_to - seg_from;
        }
        spin_unlock(&resv->lock);

        return chg;
}

/*
 * Convert the address within this vma to the page offset within
 * the mapping, huge page units here.
 */
static pgoff_t vma_hugecache_offset(struct hstate *h,
                        struct vm_area_struct *vma, unsigned long address)
{
        return linear_page_index(vma, address) >> huge_page_order(h);
}

/*
 * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
 * bits of the reservation map pointer, which are always clear due to
 * alignment.
 */
#define HPAGE_RESV_OWNER    (1UL << 0)
#define HPAGE_RESV_UNMAPPED (1UL << 1)
#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)

/*
 * These helpers are used to track how many pages are reserved for
 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
 * is guaranteed to have their future faults succeed.
 *
 * With the exception of hugetlb_dup_vma_private() which is called at fork(),
 * the reserve counters are updated with the hugetlb_lock held. It is safe
 * to reset the VMA at fork() time as it is not in use yet and there is no
 * chance of the global counters getting corrupted as a result of the values.
 *
 * The private mapping reservation is represented in a subtly different
 * manner to a shared mapping.  A shared mapping has a region map associated
 * with the underlying file, this region map represents the backing file
 * pages which have ever had a reservation assigned which this persists even
 * after the page is instantiated.  A private mapping has a region map
 * associated with the original mmap which is attached to all VMAs which
 * reference it, this region map represents those offsets which have consumed
 * reservation ie. where pages have been instantiated.
 */
static unsigned long get_vma_private_data(struct vm_area_struct *vma)
{
        return (unsigned long)vma->vm_private_data;
}

static void set_vma_private_data(struct vm_area_struct *vma,
                                                        unsigned long value)
{
        vma->vm_private_data = (void *)value;
}

static void
resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
                                          struct hugetlb_cgroup *h_cg,
                                          struct hstate *h)
{
#ifdef CONFIG_CGROUP_HUGETLB
        if (!h_cg || !h) {
                resv_map->reservation_counter = NULL;
                resv_map->pages_per_hpage = 0;
                resv_map->css = NULL;
        } else {
                resv_map->reservation_counter =
                        &h_cg->rsvd_hugepage[hstate_index(h)];
                resv_map->pages_per_hpage = pages_per_huge_page(h);
                resv_map->css = &h_cg->css;
        }
#endif
}

struct resv_map *resv_map_alloc(void)
{
        struct resv_map *resv_map = kmalloc_obj(*resv_map);
        struct file_region *rg = kmalloc_obj(*rg);

        if (!resv_map || !rg) {
                kfree(resv_map);
                kfree(rg);
                return NULL;
        }

        kref_init(&resv_map->refs);
        spin_lock_init(&resv_map->lock);
        INIT_LIST_HEAD(&resv_map->regions);
        init_rwsem(&resv_map->rw_sema);

        resv_map->adds_in_progress = 0;
        /*
         * Initialize these to 0. On shared mappings, 0's here indicate these
         * fields don't do cgroup accounting. On private mappings, these will be
         * re-initialized to the proper values, to indicate that hugetlb cgroup
         * reservations are to be un-charged from here.
         */
        resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL);

        INIT_LIST_HEAD(&resv_map->region_cache);
        list_add(&rg->link, &resv_map->region_cache);
        resv_map->region_cache_count = 1;

        return resv_map;
}

void resv_map_release(struct kref *ref)
{
        struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
        struct list_head *head = &resv_map->region_cache;
        struct file_region *rg, *trg;

        /* Clear out any active regions before we release the map. */
        region_del(resv_map, 0, LONG_MAX);

        /* ... and any entries left in the cache */
        list_for_each_entry_safe(rg, trg, head, link) {
                list_del(&rg->link);
                kfree(rg);
        }

        VM_BUG_ON(resv_map->adds_in_progress);

        kfree(resv_map);
}

static inline struct resv_map *inode_resv_map(struct inode *inode)
{
        return HUGETLBFS_I(inode)->resv_map;
}

static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
{
        VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
        if (vma->vm_flags & VM_MAYSHARE) {
                struct address_space *mapping = vma->vm_file->f_mapping;
                struct inode *inode = mapping->host;

                return inode_resv_map(inode);

        } else {
                return (struct resv_map *)(get_vma_private_data(vma) &
                                                        ~HPAGE_RESV_MASK);
        }
}

static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
{
        VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
        VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);

        set_vma_private_data(vma, (unsigned long)map);
}

static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
{
        VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
        VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);

        set_vma_private_data(vma, get_vma_private_data(vma) | flags);
}

static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
{
        VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);

        return (get_vma_private_data(vma) & flag) != 0;
}

bool __vma_private_lock(struct vm_area_struct *vma)
{
        return !(vma->vm_flags & VM_MAYSHARE) &&
                get_vma_private_data(vma) & ~HPAGE_RESV_MASK &&
                is_vma_resv_set(vma, HPAGE_RESV_OWNER);
}

void hugetlb_dup_vma_private(struct vm_area_struct *vma)
{
        VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
        /*
         * Clear vm_private_data
         * - For shared mappings this is a per-vma semaphore that may be
         *   allocated in a subsequent call to hugetlb_vm_op_open.
         *   Before clearing, make sure pointer is not associated with vma
         *   as this will leak the structure.  This is the case when called
         *   via clear_vma_resv_huge_pages() and hugetlb_vm_op_open has already
         *   been called to allocate a new structure.
         * - For MAP_PRIVATE mappings, this is the reserve map which does
         *   not apply to children.  Faults generated by the children are
         *   not guaranteed to succeed, even if read-only.
         */
        if (vma->vm_flags & VM_MAYSHARE) {
                struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

                if (vma_lock && vma_lock->vma != vma)
                        vma->vm_private_data = NULL;
        } else {
                vma->vm_private_data = NULL;
        }
}

/*
 * Reset and decrement one ref on hugepage private reservation.
 * Called with mm->mmap_lock writer semaphore held.
 * This function should be only used by mremap and operate on
 * same sized vma. It should never come here with last ref on the
 * reservation.
 */
void clear_vma_resv_huge_pages(struct vm_area_struct *vma)
{
        /*
         * Clear the old hugetlb private page reservation.
         * It has already been transferred to new_vma.
         *
         * During a mremap() operation of a hugetlb vma we call move_vma()
         * which copies vma into new_vma and unmaps vma. After the copy
         * operation both new_vma and vma share a reference to the resv_map
         * struct, and at that point vma is about to be unmapped. We don't
         * want to return the reservation to the pool at unmap of vma because
         * the reservation still lives on in new_vma, so simply decrement the
         * ref here and remove the resv_map reference from this vma.
         */
        struct resv_map *reservations = vma_resv_map(vma);

        if (reservations && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
                resv_map_put_hugetlb_cgroup_uncharge_info(reservations);
                kref_put(&reservations->refs, resv_map_release);
        }

        hugetlb_dup_vma_private(vma);
}

static void enqueue_hugetlb_folio(struct hstate *h, struct folio *folio)
{
        int nid = folio_nid(folio);

        lockdep_assert_held(&hugetlb_lock);
        VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);

        list_move(&folio->lru, &h->hugepage_freelists[nid]);
        h->free_huge_pages++;
        h->free_huge_pages_node[nid]++;
        folio_set_hugetlb_freed(folio);
}

static struct folio *dequeue_hugetlb_folio_node_exact(struct hstate *h,
                                                                int nid)
{
        struct folio *folio;
        bool pin = !!(current->flags & PF_MEMALLOC_PIN);

        lockdep_assert_held(&hugetlb_lock);
        list_for_each_entry(folio, &h->hugepage_freelists[nid], lru) {
                if (pin && !folio_is_longterm_pinnable(folio))
                        continue;

                if (folio_test_hwpoison(folio))
                        continue;

                if (is_migrate_isolate_page(&folio->page))
                        continue;

                list_move(&folio->lru, &h->hugepage_activelist);
                folio_ref_unfreeze(folio, 1);
                folio_clear_hugetlb_freed(folio);
                h->free_huge_pages--;
                h->free_huge_pages_node[nid]--;
                return folio;
        }

        return NULL;
}

static struct folio *dequeue_hugetlb_folio_nodemask(struct hstate *h, gfp_t gfp_mask,
                                                        int nid, nodemask_t *nmask)
{
        unsigned int cpuset_mems_cookie;
        struct zonelist *zonelist;
        struct zone *zone;
        struct zoneref *z;
        int node = NUMA_NO_NODE;

        /* 'nid' should not be NUMA_NO_NODE. Try to catch any misuse of it and rectifiy. */
        if (nid == NUMA_NO_NODE)
                nid = numa_node_id();

        zonelist = node_zonelist(nid, gfp_mask);

retry_cpuset:
        cpuset_mems_cookie = read_mems_allowed_begin();
        for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
                struct folio *folio;

                if (!cpuset_zone_allowed(zone, gfp_mask))
                        continue;
                /*
                 * no need to ask again on the same node. Pool is node rather than
                 * zone aware
                 */
                if (zone_to_nid(zone) == node)
                        continue;
                node = zone_to_nid(zone);

                folio = dequeue_hugetlb_folio_node_exact(h, node);
                if (folio)
                        return folio;
        }
        if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
                goto retry_cpuset;

        return NULL;
}

static unsigned long available_huge_pages(struct hstate *h)
{
        return h->free_huge_pages - h->resv_huge_pages;
}

static struct folio *dequeue_hugetlb_folio(struct hstate *h, gfp_t gfp_mask,
                                           struct mempolicy_interpreted *mpoli)
{
        nodemask_t *nodemask = mpoli->nodemask;
        struct folio *folio = NULL;

        if (mpoli->mode == MPOL_PREFERRED_MANY) {
                folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
                                                       mpoli->nid,
                                                       nodemask);

                /* Fallback to all nodes if page==NULL */
                nodemask = NULL;
        }

        if (!folio) {
                folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
                                                       mpoli->nid,
                                                       nodemask);
        }
        return folio;
}

#if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && defined(CONFIG_CONTIG_ALLOC)
static struct folio *alloc_gigantic_frozen_folio(int order, gfp_t gfp_mask,
                int nid, nodemask_t *nodemask)
{
        struct folio *folio;

        folio = hugetlb_cma_alloc_frozen_folio(order, gfp_mask, nid, nodemask);
        if (folio)
                return folio;

        if (hugetlb_cma_exclusive_alloc())
                return NULL;

        folio = (struct folio *)alloc_contig_frozen_pages(1 << order, gfp_mask,
                                                          nid, nodemask);
        return folio;
}
#else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE || !CONFIG_CONTIG_ALLOC */
static struct folio *alloc_gigantic_frozen_folio(int order, gfp_t gfp_mask, int nid,
                                          nodemask_t *nodemask)
{
        return NULL;
}
#endif

/*
 * Remove hugetlb folio from lists.
 * If vmemmap exists for the folio, clear the hugetlb flag so that the
 * folio appears as just a compound page.  Otherwise, wait until after
 * allocating vmemmap to clear the flag.
 *
 * Must be called with hugetlb lock held.
 */
void remove_hugetlb_folio(struct hstate *h, struct folio *folio,
                          bool adjust_surplus)
{
        int nid = folio_nid(folio);

        VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio(folio), folio);
        VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio);

        lockdep_assert_held(&hugetlb_lock);
        if (hstate_is_gigantic_no_runtime(h))
                return;

        list_del(&folio->lru);

        if (folio_test_hugetlb_freed(folio)) {
                folio_clear_hugetlb_freed(folio);
                h->free_huge_pages--;
                h->free_huge_pages_node[nid]--;
        }
        if (adjust_surplus) {
                h->surplus_huge_pages--;
                h->surplus_huge_pages_node[nid]--;
        }

        /*
         * We can only clear the hugetlb flag after allocating vmemmap
         * pages.  Otherwise, someone (memory error handling) may try to write
         * to tail struct pages.
         */
        if (!folio_test_hugetlb_vmemmap_optimized(folio))
                __folio_clear_hugetlb(folio);

        h->nr_huge_pages--;
        h->nr_huge_pages_node[nid]--;
}

void add_hugetlb_folio(struct hstate *h, struct folio *folio,
                       bool adjust_surplus)
{
        int nid = folio_nid(folio);

        VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio);

        lockdep_assert_held(&hugetlb_lock);

        INIT_LIST_HEAD(&folio->lru);
        h->nr_huge_pages++;
        h->nr_huge_pages_node[nid]++;

        if (adjust_surplus) {
                h->surplus_huge_pages++;
                h->surplus_huge_pages_node[nid]++;
        }

        __folio_set_hugetlb(folio);
        folio_change_private(folio, NULL);
        /*
         * We have to set hugetlb_vmemmap_optimized again as above
         * folio_change_private(folio, NULL) cleared it.
         */
        folio_set_hugetlb_vmemmap_optimized(folio);

        arch_clear_hugetlb_flags(folio);
        enqueue_hugetlb_folio(h, folio);
}

static void __update_and_free_hugetlb_folio(struct hstate *h,
                                                struct folio *folio)
{
        bool clear_flag = folio_test_hugetlb_vmemmap_optimized(folio);

        if (hstate_is_gigantic_no_runtime(h))
                return;

        /*
         * If we don't know which subpages are hwpoisoned, we can't free
         * the hugepage, so it's leaked intentionally.
         */
        if (folio_test_hugetlb_raw_hwp_unreliable(folio))
                return;

        /*
         * If folio is not vmemmap optimized (!clear_flag), then the folio
         * is no longer identified as a hugetlb page.  hugetlb_vmemmap_restore_folio
         * can only be passed hugetlb pages and will BUG otherwise.
         */
        if (clear_flag && hugetlb_vmemmap_restore_folio(h, folio)) {
                spin_lock_irq(&hugetlb_lock);
                /*
                 * If we cannot allocate vmemmap pages, just refuse to free the
                 * page and put the page back on the hugetlb free list and treat
                 * as a surplus page.
                 */
                add_hugetlb_folio(h, folio, true);
                spin_unlock_irq(&hugetlb_lock);
                return;
        }

        /*
         * If vmemmap pages were allocated above, then we need to clear the
         * hugetlb flag under the hugetlb lock.
         */
        if (folio_test_hugetlb(folio)) {
                spin_lock_irq(&hugetlb_lock);
                __folio_clear_hugetlb(folio);
                spin_unlock_irq(&hugetlb_lock);
        }

        /*
         * Move PageHWPoison flag from head page to the raw error pages,
         * which makes any healthy subpages reusable.
         */
        if (unlikely(folio_test_hwpoison(folio)))
                folio_clear_hugetlb_hwpoison(folio);

        VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
        if (folio_test_hugetlb_cma(folio))
                hugetlb_cma_free_frozen_folio(folio);
        else
                free_frozen_pages(&folio->page, folio_order(folio));
}

/*
 * As update_and_free_hugetlb_folio() can be called under any context, so we cannot
 * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the
 * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate
 * the vmemmap pages.
 *
 * free_hpage_workfn() locklessly retrieves the linked list of pages to be
 * freed and frees them one-by-one. As the page->mapping pointer is going
 * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node
 * structure of a lockless linked list of huge pages to be freed.
 */
static LLIST_HEAD(hpage_freelist);

static void free_hpage_workfn(struct work_struct *work)
{
        struct llist_node *node;

        node = llist_del_all(&hpage_freelist);

        while (node) {
                struct folio *folio;
                struct hstate *h;

                folio = container_of((struct address_space **)node,
                                     struct folio, mapping);
                node = node->next;
                folio->mapping = NULL;
                /*
                 * The VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio) in
                 * folio_hstate() is going to trigger because a previous call to
                 * remove_hugetlb_folio() will clear the hugetlb bit, so do
                 * not use folio_hstate() directly.
                 */
                h = size_to_hstate(folio_size(folio));

                __update_and_free_hugetlb_folio(h, folio);

                cond_resched();
        }
}
static DECLARE_WORK(free_hpage_work, free_hpage_workfn);

static inline void flush_free_hpage_work(struct hstate *h)
{
        if (hugetlb_vmemmap_optimizable(h))
                flush_work(&free_hpage_work);
}

static void update_and_free_hugetlb_folio(struct hstate *h, struct folio *folio,
                                 bool atomic)
{
        if (!folio_test_hugetlb_vmemmap_optimized(folio) || !atomic) {
                __update_and_free_hugetlb_folio(h, folio);
                return;
        }

        /*
         * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages.
         *
         * Only call schedule_work() if hpage_freelist is previously
         * empty. Otherwise, schedule_work() had been called but the workfn
         * hasn't retrieved the list yet.
         */
        if (llist_add((struct llist_node *)&folio->mapping, &hpage_freelist))
                schedule_work(&free_hpage_work);
}

static void bulk_vmemmap_restore_error(struct hstate *h,
                                        struct list_head *folio_list,
                                        struct list_head *non_hvo_folios)
{
        struct folio *folio, *t_folio;

        if (!list_empty(non_hvo_folios)) {
                /*
                 * Free any restored hugetlb pages so that restore of the
                 * entire list can be retried.
                 * The idea is that in the common case of ENOMEM errors freeing
                 * hugetlb pages with vmemmap we will free up memory so that we
                 * can allocate vmemmap for more hugetlb pages.
                 */
                list_for_each_entry_safe(folio, t_folio, non_hvo_folios, lru) {
                        list_del(&folio->lru);
                        spin_lock_irq(&hugetlb_lock);
                        __folio_clear_hugetlb(folio);
                        spin_unlock_irq(&hugetlb_lock);
                        update_and_free_hugetlb_folio(h, folio, false);
                        cond_resched();
                }
        } else {
                /*
                 * In the case where there are no folios which can be
                 * immediately freed, we loop through the list trying to restore
                 * vmemmap individually in the hope that someone elsewhere may
                 * have done something to cause success (such as freeing some
                 * memory).  If unable to restore a hugetlb page, the hugetlb
                 * page is made a surplus page and removed from the list.
                 * If are able to restore vmemmap and free one hugetlb page, we
                 * quit processing the list to retry the bulk operation.
                 */
                list_for_each_entry_safe(folio, t_folio, folio_list, lru)
                        if (hugetlb_vmemmap_restore_folio(h, folio)) {
                                list_del(&folio->lru);
                                spin_lock_irq(&hugetlb_lock);
                                add_hugetlb_folio(h, folio, true);
                                spin_unlock_irq(&hugetlb_lock);
                        } else {
                                list_del(&folio->lru);
                                spin_lock_irq(&hugetlb_lock);
                                __folio_clear_hugetlb(folio);
                                spin_unlock_irq(&hugetlb_lock);
                                update_and_free_hugetlb_folio(h, folio, false);
                                cond_resched();
                                break;
                        }
        }
}

static void update_and_free_pages_bulk(struct hstate *h,
                                                struct list_head *folio_list)
{
        long ret;
        struct folio *folio, *t_folio;
        LIST_HEAD(non_hvo_folios);

        /*
         * First allocate required vmemmmap (if necessary) for all folios.
         * Carefully handle errors and free up any available hugetlb pages
         * in an effort to make forward progress.
         */
retry:
        ret = hugetlb_vmemmap_restore_folios(h, folio_list, &non_hvo_folios);
        if (ret < 0) {
                bulk_vmemmap_restore_error(h, folio_list, &non_hvo_folios);
                goto retry;
        }

        /*
         * At this point, list should be empty, ret should be >= 0 and there
         * should only be pages on the non_hvo_folios list.
         * Do note that the non_hvo_folios list could be empty.
         * Without HVO enabled, ret will be 0 and there is no need to call
         * __folio_clear_hugetlb as this was done previously.
         */
        VM_WARN_ON(!list_empty(folio_list));
        VM_WARN_ON(ret < 0);
        if (!list_empty(&non_hvo_folios) && ret) {
                spin_lock_irq(&hugetlb_lock);
                list_for_each_entry(folio, &non_hvo_folios, lru)
                        __folio_clear_hugetlb(folio);
                spin_unlock_irq(&hugetlb_lock);
        }

        list_for_each_entry_safe(folio, t_folio, &non_hvo_folios, lru) {
                update_and_free_hugetlb_folio(h, folio, false);
                cond_resched();
        }
}

struct hstate *size_to_hstate(unsigned long size)
{
        struct hstate *h;

        for_each_hstate(h) {
                if (huge_page_size(h) == size)
                        return h;
        }
        return NULL;
}

void free_huge_folio(struct folio *folio)
{
        /*
         * Can't pass hstate in here because it is called from the
         * generic mm code.
         */
        struct hstate *h = folio_hstate(folio);
        int nid = folio_nid(folio);
        struct hugepage_subpool *spool = hugetlb_folio_subpool(folio);
        bool restore_reserve;
        unsigned long flags;

        VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
        VM_BUG_ON_FOLIO(folio_mapcount(folio), folio);

        hugetlb_set_folio_subpool(folio, NULL);
        if (folio_test_anon(folio))
                __ClearPageAnonExclusive(&folio->page);
        folio->mapping = NULL;
        restore_reserve = folio_test_hugetlb_restore_reserve(folio);
        folio_clear_hugetlb_restore_reserve(folio);

        /*
         * If HPageRestoreReserve was set on page, page allocation consumed a
         * reservation.  If the page was associated with a subpool, there
         * would have been a page reserved in the subpool before allocation
         * via hugepage_subpool_get_pages().  Since we are 'restoring' the
         * reservation, do not call hugepage_subpool_put_pages() as this will
         * remove the reserved page from the subpool.
         */
        if (!restore_reserve) {
                /*
                 * A return code of zero implies that the subpool will be
                 * under its minimum size if the reservation is not restored
                 * after page is free.  Therefore, force restore_reserve
                 * operation.
                 */
                if (hugepage_subpool_put_pages(spool, 1) == 0)
                        restore_reserve = true;
        }

        spin_lock_irqsave(&hugetlb_lock, flags);
        folio_clear_hugetlb_migratable(folio);
        hugetlb_cgroup_uncharge_folio(hstate_index(h),
                                     pages_per_huge_page(h), folio);
        hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h),
                                          pages_per_huge_page(h), folio);
        lruvec_stat_mod_folio(folio, NR_HUGETLB, -pages_per_huge_page(h));
        mem_cgroup_uncharge(folio);
        if (restore_reserve)
                h->resv_huge_pages++;

        if (folio_test_hugetlb_temporary(folio)) {
                remove_hugetlb_folio(h, folio, false);
                spin_unlock_irqrestore(&hugetlb_lock, flags);
                update_and_free_hugetlb_folio(h, folio, true);
        } else if (h->surplus_huge_pages_node[nid]) {
                /* remove the page from active list */
                remove_hugetlb_folio(h, folio, true);
                spin_unlock_irqrestore(&hugetlb_lock, flags);
                update_and_free_hugetlb_folio(h, folio, true);
        } else {
                arch_clear_hugetlb_flags(folio);
                enqueue_hugetlb_folio(h, folio);
                spin_unlock_irqrestore(&hugetlb_lock, flags);
        }
}

/*
 * Must be called with the hugetlb lock held
 */
static void account_new_hugetlb_folio(struct hstate *h, struct folio *folio)
{
        lockdep_assert_held(&hugetlb_lock);
        h->nr_huge_pages++;
        h->nr_huge_pages_node[folio_nid(folio)]++;
}

void init_new_hugetlb_folio(struct folio *folio)
{
        __folio_set_hugetlb(folio);
        INIT_LIST_HEAD(&folio->lru);
        hugetlb_set_folio_subpool(folio, NULL);
        set_hugetlb_cgroup(folio, NULL);
        set_hugetlb_cgroup_rsvd(folio, NULL);
}

/*
 * Find and lock address space (mapping) in write mode.
 *
 * Upon entry, the folio is locked which means that folio_mapping() is
 * stable.  Due to locking order, we can only trylock_write.  If we can
 * not get the lock, simply return NULL to caller.
 */
struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio)
{
        struct address_space *mapping = folio_mapping(folio);

        if (!mapping)
                return mapping;

        if (i_mmap_trylock_write(mapping))
                return mapping;

        return NULL;
}

static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask,
                int nid, nodemask_t *nmask, nodemask_t *node_alloc_noretry)
{
        struct folio *folio;
        bool alloc_try_hard = true;

        /*
         * By default we always try hard to allocate the folio with
         * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating folios in
         * a loop (to adjust global huge page counts) and previous allocation
         * failed, do not continue to try hard on the same node.  Use the
         * node_alloc_noretry bitmap to manage this state information.
         */
        if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
                alloc_try_hard = false;
        if (alloc_try_hard)
                gfp_mask |= __GFP_RETRY_MAYFAIL;

        folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask,
                                                     ALLOC_DEFAULT);

        /*
         * If we did not specify __GFP_RETRY_MAYFAIL, but still got a
         * folio this indicates an overall state change.  Clear bit so
         * that we resume normal 'try hard' allocations.
         */
        if (node_alloc_noretry && folio && !alloc_try_hard)
                node_clear(nid, *node_alloc_noretry);

        /*
         * If we tried hard to get a folio but failed, set bit so that
         * subsequent attempts will not try as hard until there is an
         * overall state change.
         */
        if (node_alloc_noretry && !folio && alloc_try_hard)
                node_set(nid, *node_alloc_noretry);

        if (!folio) {
                __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
                return NULL;
        }

        __count_vm_event(HTLB_BUDDY_PGALLOC);
        return folio;
}

static struct folio *only_alloc_fresh_hugetlb_folio(struct hstate *h,
                gfp_t gfp_mask, int nid, nodemask_t *nmask,
                nodemask_t *node_alloc_noretry)
{
        struct folio *folio;
        int order = huge_page_order(h);

        if (nid == NUMA_NO_NODE)
                nid = numa_mem_id();

        if (order_is_gigantic(order))
                folio = alloc_gigantic_frozen_folio(order, gfp_mask, nid, nmask);
        else
                folio = alloc_buddy_frozen_folio(order, gfp_mask, nid, nmask,
                                                 node_alloc_noretry);
        if (folio)
                init_new_hugetlb_folio(folio);
        return folio;
}

/*
 * Common helper to allocate a fresh hugetlb folio. All specific allocators
 * should use this function to get new hugetlb folio
 *
 * Note that returned folio is 'frozen':  ref count of head page and all tail
 * pages is zero, and the accounting must be done in the caller.
 */
static struct folio *alloc_fresh_hugetlb_folio(struct hstate *h,
                gfp_t gfp_mask, int nid, nodemask_t *nmask)
{
        struct folio *folio;

        folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
        if (folio)
                hugetlb_vmemmap_optimize_folio(h, folio);
        return folio;
}

void prep_and_add_allocated_folios(struct hstate *h,
                                   struct list_head *folio_list)
{
        unsigned long flags;
        struct folio *folio, *tmp_f;

        /* Send list for bulk vmemmap optimization processing */
        hugetlb_vmemmap_optimize_folios(h, folio_list);

        /* Add all new pool pages to free lists in one lock cycle */
        spin_lock_irqsave(&hugetlb_lock, flags);
        list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
                account_new_hugetlb_folio(h, folio);
                enqueue_hugetlb_folio(h, folio);
        }
        spin_unlock_irqrestore(&hugetlb_lock, flags);
}

/*
 * Allocates a fresh hugetlb page in a node interleaved manner.  The page
 * will later be added to the appropriate hugetlb pool.
 */
static struct folio *alloc_pool_huge_folio(struct hstate *h,
                                        nodemask_t *nodes_allowed,
                                        nodemask_t *node_alloc_noretry,
                                        int *next_node)
{
        gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
        int nr_nodes, node;

        for_each_node_mask_to_alloc(next_node, nr_nodes, node, nodes_allowed) {
                struct folio *folio;

                folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, node,
                                        nodes_allowed, node_alloc_noretry);
                if (folio)
                        return folio;
        }

        return NULL;
}

/*
 * Remove huge page from pool from next node to free.  Attempt to keep
 * persistent huge pages more or less balanced over allowed nodes.
 * This routine only 'removes' the hugetlb page.  The caller must make
 * an additional call to free the page to low level allocators.
 * Called with hugetlb_lock locked.
 */
static struct folio *remove_pool_hugetlb_folio(struct hstate *h,
                nodemask_t *nodes_allowed, bool acct_surplus)
{
        int nr_nodes, node;
        struct folio *folio = NULL;

        lockdep_assert_held(&hugetlb_lock);
        for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
                /*
                 * If we're returning unused surplus pages, only examine
                 * nodes with surplus pages.
                 */
                if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
                    !list_empty(&h->hugepage_freelists[node])) {
                        folio = list_entry(h->hugepage_freelists[node].next,
                                          struct folio, lru);
                        remove_hugetlb_folio(h, folio, acct_surplus);
                        break;
                }
        }

        return folio;
}

/*
 * Dissolve a given free hugetlb folio into free buddy pages. This function
 * does nothing for in-use hugetlb folios and non-hugetlb folios.
 * This function returns values like below:
 *
 *  -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages
 *           when the system is under memory pressure and the feature of
 *           freeing unused vmemmap pages associated with each hugetlb page
 *           is enabled.
 *  -EBUSY:  failed to dissolved free hugepages or the hugepage is in-use
 *           (allocated or reserved.)
 *       0:  successfully dissolved free hugepages or the page is not a
 *           hugepage (considered as already dissolved)
 */
int dissolve_free_hugetlb_folio(struct folio *folio)
{
        int rc = -EBUSY;

retry:
        /* Not to disrupt normal path by vainly holding hugetlb_lock */
        if (!folio_test_hugetlb(folio))
                return 0;

        spin_lock_irq(&hugetlb_lock);
        if (!folio_test_hugetlb(folio)) {
                rc = 0;
                goto out;
        }

        if (!folio_ref_count(folio)) {
                struct hstate *h = folio_hstate(folio);
                bool adjust_surplus = false;

                if (!available_huge_pages(h))
                        goto out;

                /*
                 * We should make sure that the page is already on the free list
                 * when it is dissolved.
                 */
                if (unlikely(!folio_test_hugetlb_freed(folio))) {
                        spin_unlock_irq(&hugetlb_lock);
                        cond_resched();

                        /*
                         * Theoretically, we should return -EBUSY when we
                         * encounter this race. In fact, we have a chance
                         * to successfully dissolve the page if we do a
                         * retry. Because the race window is quite small.
                         * If we seize this opportunity, it is an optimization
                         * for increasing the success rate of dissolving page.
                         */
                        goto retry;
                }

                if (h->surplus_huge_pages_node[folio_nid(folio)])
                        adjust_surplus = true;
                remove_hugetlb_folio(h, folio, adjust_surplus);
                h->max_huge_pages--;
                spin_unlock_irq(&hugetlb_lock);

                /*
                 * Normally update_and_free_hugtlb_folio will allocate required vmemmmap
                 * before freeing the page.  update_and_free_hugtlb_folio will fail to
                 * free the page if it can not allocate required vmemmap.  We
                 * need to adjust max_huge_pages if the page is not freed.
                 * Attempt to allocate vmemmmap here so that we can take
                 * appropriate action on failure.
                 *
                 * The folio_test_hugetlb check here is because
                 * remove_hugetlb_folio will clear hugetlb folio flag for
                 * non-vmemmap optimized hugetlb folios.
                 */
                if (folio_test_hugetlb(folio)) {
                        rc = hugetlb_vmemmap_restore_folio(h, folio);
                        if (rc) {
                                spin_lock_irq(&hugetlb_lock);
                                add_hugetlb_folio(h, folio, adjust_surplus);
                                h->max_huge_pages++;
                                goto out;
                        }
                } else {
                        rc = 0;
                }

                update_and_free_hugetlb_folio(h, folio, false);
                return rc;
        }
out:
        spin_unlock_irq(&hugetlb_lock);
        return rc;
}

/*
 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
 * make specified memory blocks removable from the system.
 * Note that this will dissolve a free gigantic hugepage completely, if any
 * part of it lies within the given range.
 * Also note that if dissolve_free_hugetlb_folio() returns with an error, all
 * free hugetlb folios that were dissolved before that error are lost.
 */
int dissolve_free_hugetlb_folios(unsigned long start_pfn, unsigned long end_pfn)
{
        unsigned long pfn;
        struct folio *folio;
        int rc = 0;
        unsigned int order;
        struct hstate *h;

        if (!hugepages_supported())
                return rc;

        order = huge_page_order(&default_hstate);
        for_each_hstate(h)
                order = min(order, huge_page_order(h));

        for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order) {
                folio = pfn_folio(pfn);
                rc = dissolve_free_hugetlb_folio(folio);
                if (rc)
                        break;
        }

        return rc;
}

/*
 * Allocates a fresh surplus page from the page allocator.
 */
static struct folio *alloc_surplus_hugetlb_folio(struct hstate *h,
                                gfp_t gfp_mask,        int nid, nodemask_t *nmask)
{
        struct folio *folio = NULL;

        if (hstate_is_gigantic_no_runtime(h))
                return NULL;

        spin_lock_irq(&hugetlb_lock);
        if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
                goto out_unlock;
        spin_unlock_irq(&hugetlb_lock);

        folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask);
        if (!folio)
                return NULL;

        spin_lock_irq(&hugetlb_lock);
        /*
         * nr_huge_pages needs to be adjusted within the same lock cycle
         * as surplus_pages, otherwise it might confuse
         * persistent_huge_pages() momentarily.
         */
        account_new_hugetlb_folio(h, folio);

        /*
         * We could have raced with the pool size change.
         * Double check that and simply deallocate the new page
         * if we would end up overcommiting the surpluses. Abuse
         * temporary page to workaround the nasty free_huge_folio
         * codeflow
         */
        if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
                folio_set_hugetlb_temporary(folio);
                spin_unlock_irq(&hugetlb_lock);
                free_huge_folio(folio);
                return NULL;
        }

        h->surplus_huge_pages++;
        h->surplus_huge_pages_node[folio_nid(folio)]++;

out_unlock:
        spin_unlock_irq(&hugetlb_lock);

        return folio;
}

static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mask,
                                     int nid, nodemask_t *nmask)
{
        struct folio *folio;

        if (hstate_is_gigantic(h))
                return NULL;

        folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask);
        if (!folio)
                return NULL;

        spin_lock_irq(&hugetlb_lock);
        account_new_hugetlb_folio(h, folio);
        spin_unlock_irq(&hugetlb_lock);

        /* fresh huge pages are frozen */
        folio_ref_unfreeze(folio, 1);
        /*
         * We do not account these pages as surplus because they are only
         * temporary and will be released properly on the last reference
         */
        folio_set_hugetlb_temporary(folio);

        return folio;
}

static
struct folio *alloc_buddy_hugetlb_folio(struct hstate *h,
                gfp_t gfp_mask, struct mempolicy_interpreted *mpoli)
{
        struct folio *folio = NULL;
        nodemask_t *nodemask = mpoli->nodemask;

        if (mpoli->mode == MPOL_PREFERRED_MANY) {
                gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);

                folio = alloc_surplus_hugetlb_folio(h, gfp, mpoli->nid,
                                                    nodemask);

                /* Fallback to all nodes if page==NULL */
                nodemask = NULL;
        }

        if (!folio) {
                folio = alloc_surplus_hugetlb_folio(h, gfp_mask, mpoli->nid,
                                                    nodemask);
        }

        return folio;
}

struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
                nodemask_t *nmask, gfp_t gfp_mask)
{
        struct folio *folio;

        spin_lock_irq(&hugetlb_lock);
        if (!h->resv_huge_pages) {
                spin_unlock_irq(&hugetlb_lock);
                return NULL;
        }

        folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, preferred_nid,
                                               nmask);
        if (folio)
                h->resv_huge_pages--;

        spin_unlock_irq(&hugetlb_lock);
        return folio;
}

/* folio migration callback function */
struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
                nodemask_t *nmask, gfp_t gfp_mask, bool allow_alloc_fallback)
{
        spin_lock_irq(&hugetlb_lock);
        if (available_huge_pages(h)) {
                struct folio *folio;

                folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
                                                preferred_nid, nmask);
                if (folio) {
                        spin_unlock_irq(&hugetlb_lock);
                        return folio;
                }
        }
        spin_unlock_irq(&hugetlb_lock);

        /* We cannot fallback to other nodes, as we could break the per-node pool. */
        if (!allow_alloc_fallback)
                gfp_mask |= __GFP_THISNODE;

        return alloc_migrate_hugetlb_folio(h, gfp_mask, preferred_nid, nmask);
}

static nodemask_t *policy_mbind_nodemask(gfp_t gfp)
{
#ifdef CONFIG_NUMA
        struct mempolicy *mpol = get_task_policy(current);

        /*
         * Only enforce MPOL_BIND policy which overlaps with cpuset policy
         * (from policy_nodemask) specifically for hugetlb case
         */
        if (mpol->mode == MPOL_BIND &&
                (apply_policy_zone(mpol, gfp_zone(gfp)) &&
                 cpuset_nodemask_valid_mems_allowed(&mpol->nodes)))
                return &mpol->nodes;
#endif
        return NULL;
}

/*
 * Increase the hugetlb pool such that it can accommodate a reservation
 * of size 'delta'.
 */
static int gather_surplus_pages(struct hstate *h, long delta)
        __must_hold(&hugetlb_lock)
{
        LIST_HEAD(surplus_list);
        struct folio *folio, *tmp;
        int ret;
        long i;
        long needed, allocated;
        bool alloc_ok = true;
        nodemask_t *mbind_nodemask, alloc_nodemask;

        mbind_nodemask = policy_mbind_nodemask(htlb_alloc_mask(h));
        if (mbind_nodemask)
                nodes_and(alloc_nodemask, *mbind_nodemask, cpuset_current_mems_allowed);
        else
                alloc_nodemask = cpuset_current_mems_allowed;

        lockdep_assert_held(&hugetlb_lock);
        needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
        if (needed <= 0) {
                h->resv_huge_pages += delta;
                return 0;
        }

        allocated = 0;

        ret = -ENOMEM;
retry:
        spin_unlock_irq(&hugetlb_lock);
        for (i = 0; i < needed; i++) {
                folio = NULL;

                /*
                 * It is okay to use NUMA_NO_NODE because we use numa_mem_id()
                 * down the road to pick the current node if that is the case.
                 */
                folio = alloc_surplus_hugetlb_folio(h, htlb_alloc_mask(h),
                                                    NUMA_NO_NODE, &alloc_nodemask);
                if (!folio) {
                        alloc_ok = false;
                        break;
                }
                list_add(&folio->lru, &surplus_list);
                cond_resched();
        }
        allocated += i;

        /*
         * After retaking hugetlb_lock, we need to recalculate 'needed'
         * because either resv_huge_pages or free_huge_pages may have changed.
         */
        spin_lock_irq(&hugetlb_lock);
        needed = (h->resv_huge_pages + delta) -
                        (h->free_huge_pages + allocated);
        if (needed > 0) {
                if (alloc_ok)
                        goto retry;
                /*
                 * We were not able to allocate enough pages to
                 * satisfy the entire reservation so we free what
                 * we've allocated so far.
                 */
                goto free;
        }
        /*
         * The surplus_list now contains _at_least_ the number of extra pages
         * needed to accommodate the reservation.  Add the appropriate number
         * of pages to the hugetlb pool and free the extras back to the buddy
         * allocator.  Commit the entire reservation here to prevent another
         * process from stealing the pages as they are added to the pool but
         * before they are reserved.
         */
        needed += allocated;
        h->resv_huge_pages += delta;
        ret = 0;

        /* Free the needed pages to the hugetlb pool */
        list_for_each_entry_safe(folio, tmp, &surplus_list, lru) {
                if ((--needed) < 0)
                        break;
                /* Add the page to the hugetlb allocator */
                enqueue_hugetlb_folio(h, folio);
        }
free:
        spin_unlock_irq(&hugetlb_lock);

        /*
         * Free unnecessary surplus pages to the buddy allocator.
         * Pages have no ref count, call free_huge_folio directly.
         */
        list_for_each_entry_safe(folio, tmp, &surplus_list, lru)
                free_huge_folio(folio);
        spin_lock_irq(&hugetlb_lock);

        return ret;
}

/*
 * This routine has two main purposes:
 * 1) Decrement the reservation count (resv_huge_pages) by the value passed
 *    in unused_resv_pages.  This corresponds to the prior adjustments made
 *    to the associated reservation map.
 * 2) Free any unused surplus pages that may have been allocated to satisfy
 *    the reservation.  As many as unused_resv_pages may be freed.
 */
static void return_unused_surplus_pages(struct hstate *h,
                                        unsigned long unused_resv_pages)
{
        unsigned long nr_pages;
        LIST_HEAD(page_list);

        lockdep_assert_held(&hugetlb_lock);
        /* Uncommit the reservation */
        h->resv_huge_pages -= unused_resv_pages;

        if (hstate_is_gigantic_no_runtime(h))
                goto out;

        /*
         * Part (or even all) of the reservation could have been backed
         * by pre-allocated pages. Only free surplus pages.
         */
        nr_pages = min(unused_resv_pages, h->surplus_huge_pages);

        /*
         * We want to release as many surplus pages as possible, spread
         * evenly across all nodes with memory. Iterate across these nodes
         * until we can no longer free unreserved surplus pages. This occurs
         * when the nodes with surplus pages have no free pages.
         * remove_pool_hugetlb_folio() will balance the freed pages across the
         * on-line nodes with memory and will handle the hstate accounting.
         */
        while (nr_pages--) {
                struct folio *folio;

                folio = remove_pool_hugetlb_folio(h, &node_states[N_MEMORY], 1);
                if (!folio)
                        goto out;

                list_add(&folio->lru, &page_list);
        }

out:
        spin_unlock_irq(&hugetlb_lock);
        update_and_free_pages_bulk(h, &page_list);
        spin_lock_irq(&hugetlb_lock);
}


/*
 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
 * are used by the huge page allocation routines to manage reservations.
 *
 * vma_needs_reservation is called to determine if the huge page at addr
 * within the vma has an associated reservation.  If a reservation is
 * needed, the value 1 is returned.  The caller is then responsible for
 * managing the global reservation and subpool usage counts.  After
 * the huge page has been allocated, vma_commit_reservation is called
 * to add the page to the reservation map.  If the page allocation fails,
 * the reservation must be ended instead of committed.  vma_end_reservation
 * is called in such cases.
 *
 * In the normal case, vma_commit_reservation returns the same value
 * as the preceding vma_needs_reservation call.  The only time this
 * is not the case is if a reserve map was changed between calls.  It
 * is the responsibility of the caller to notice the difference and
 * take appropriate action.
 *
 * vma_add_reservation is used in error paths where a reservation must
 * be restored when a newly allocated huge page must be freed.  It is
 * to be called after calling vma_needs_reservation to determine if a
 * reservation exists.
 *
 * vma_del_reservation is used in error paths where an entry in the reserve
 * map was created during huge page allocation and must be removed.  It is to
 * be called after calling vma_needs_reservation to determine if a reservation
 * exists.
 */
enum vma_resv_mode {
        VMA_NEEDS_RESV,
        VMA_COMMIT_RESV,
        VMA_END_RESV,
        VMA_ADD_RESV,
        VMA_DEL_RESV,
};
static long __vma_reservation_common(struct hstate *h,
                                struct vm_area_struct *vma, unsigned long addr,
                                enum vma_resv_mode mode)
{
        struct resv_map *resv;
        pgoff_t idx;
        long ret;
        long dummy_out_regions_needed;

        resv = vma_resv_map(vma);
        if (!resv)
                return 1;

        idx = vma_hugecache_offset(h, vma, addr);
        switch (mode) {
        case VMA_NEEDS_RESV:
                ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);
                /* We assume that vma_reservation_* routines always operate on
                 * 1 page, and that adding to resv map a 1 page entry can only
                 * ever require 1 region.
                 */
                VM_BUG_ON(dummy_out_regions_needed != 1);
                break;
        case VMA_COMMIT_RESV:
                ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
                /* region_add calls of range 1 should never fail. */
                VM_BUG_ON(ret < 0);
                break;
        case VMA_END_RESV:
                region_abort(resv, idx, idx + 1, 1);
                ret = 0;
                break;
        case VMA_ADD_RESV:
                if (vma->vm_flags & VM_MAYSHARE) {
                        ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
                        /* region_add calls of range 1 should never fail. */
                        VM_BUG_ON(ret < 0);
                } else {
                        region_abort(resv, idx, idx + 1, 1);
                        ret = region_del(resv, idx, idx + 1);
                }
                break;
        case VMA_DEL_RESV:
                if (vma->vm_flags & VM_MAYSHARE) {
                        region_abort(resv, idx, idx + 1, 1);
                        ret = region_del(resv, idx, idx + 1);
                } else {
                        ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
                        /* region_add calls of range 1 should never fail. */
                        VM_BUG_ON(ret < 0);
                }
                break;
        default:
                BUG();
        }

        if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV)
                return ret;
        /*
         * We know private mapping must have HPAGE_RESV_OWNER set.
         *
         * In most cases, reserves always exist for private mappings.
         * However, a file associated with mapping could have been
         * hole punched or truncated after reserves were consumed.
         * As subsequent fault on such a range will not use reserves.
         * Subtle - The reserve map for private mappings has the
         * opposite meaning than that of shared mappings.  If NO
         * entry is in the reserve map, it means a reservation exists.
         * If an entry exists in the reserve map, it means the
         * reservation has already been consumed.  As a result, the
         * return value of this routine is the opposite of the
         * value returned from reserve map manipulation routines above.
         */
        if (ret > 0)
                return 0;
        if (ret == 0)
                return 1;
        return ret;
}

static long vma_needs_reservation(struct hstate *h,
                        struct vm_area_struct *vma, unsigned long addr)
{
        return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
}

static long vma_commit_reservation(struct hstate *h,
                        struct vm_area_struct *vma, unsigned long addr)
{
        return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
}

static void vma_end_reservation(struct hstate *h,
                        struct vm_area_struct *vma, unsigned long addr)
{
        (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
}

static long vma_add_reservation(struct hstate *h,
                        struct vm_area_struct *vma, unsigned long addr)
{
        return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
}

static long vma_del_reservation(struct hstate *h,
                        struct vm_area_struct *vma, unsigned long addr)
{
        return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV);
}

/*
 * This routine is called to restore reservation information on error paths.
 * It should ONLY be called for folios allocated via alloc_hugetlb_folio(),
 * and the hugetlb mutex should remain held when calling this routine.
 *
 * It handles two specific cases:
 * 1) A reservation was in place and the folio consumed the reservation.
 *    hugetlb_restore_reserve is set in the folio.
 * 2) No reservation was in place for the page, so hugetlb_restore_reserve is
 *    not set.  However, alloc_hugetlb_folio always updates the reserve map.
 *
 * In case 1, free_huge_folio later in the error path will increment the
 * global reserve count.  But, free_huge_folio does not have enough context
 * to adjust the reservation map.  This case deals primarily with private
 * mappings.  Adjust the reserve map here to be consistent with global
 * reserve count adjustments to be made by free_huge_folio.  Make sure the
 * reserve map indicates there is a reservation present.
 *
 * In case 2, simply undo reserve map modifications done by alloc_hugetlb_folio.
 */
void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
                        unsigned long address, struct folio *folio)
{
        long rc = vma_needs_reservation(h, vma, address);

        if (folio_test_hugetlb_restore_reserve(folio)) {
                if (unlikely(rc < 0))
                        /*
                         * Rare out of memory condition in reserve map
                         * manipulation.  Clear hugetlb_restore_reserve so
                         * that global reserve count will not be incremented
                         * by free_huge_folio.  This will make it appear
                         * as though the reservation for this folio was
                         * consumed.  This may prevent the task from
                         * faulting in the folio at a later time.  This
                         * is better than inconsistent global huge page
                         * accounting of reserve counts.
                         */
                        folio_clear_hugetlb_restore_reserve(folio);
                else if (rc)
                        (void)vma_add_reservation(h, vma, address);
                else
                        vma_end_reservation(h, vma, address);
        } else {
                if (!rc) {
                        /*
                         * This indicates there is an entry in the reserve map
                         * not added by alloc_hugetlb_folio.  We know it was added
                         * before the alloc_hugetlb_folio call, otherwise
                         * hugetlb_restore_reserve would be set on the folio.
                         * Remove the entry so that a subsequent allocation
                         * does not consume a reservation.
                         */
                        rc = vma_del_reservation(h, vma, address);
                        if (rc < 0)
                                /*
                                 * VERY rare out of memory condition.  Since
                                 * we can not delete the entry, set
                                 * hugetlb_restore_reserve so that the reserve
                                 * count will be incremented when the folio
                                 * is freed.  This reserve will be consumed
                                 * on a subsequent allocation.
                                 */
                                folio_set_hugetlb_restore_reserve(folio);
                } else if (rc < 0) {
                        /*
                         * Rare out of memory condition from
                         * vma_needs_reservation call.  Memory allocation is
                         * only attempted if a new entry is needed.  Therefore,
                         * this implies there is not an entry in the
                         * reserve map.
                         *
                         * For shared mappings, no entry in the map indicates
                         * no reservation.  We are done.
                         */
                        if (!(vma->vm_flags & VM_MAYSHARE))
                                /*
                                 * For private mappings, no entry indicates
                                 * a reservation is present.  Since we can
                                 * not add an entry, set hugetlb_restore_reserve
                                 * on the folio so reserve count will be
                                 * incremented when freed.  This reserve will
                                 * be consumed on a subsequent allocation.
                                 */
                                folio_set_hugetlb_restore_reserve(folio);
                } else {
                        /*
                         * No reservation present, do nothing
                         */
                        vma_end_reservation(h, vma, address);
                }
        }
}

/*
 * alloc_and_dissolve_hugetlb_folio - Allocate a new folio and dissolve
 * the old one
 * @old_folio: Old folio to dissolve
 * @list: List to isolate the page in case we need to
 * Returns 0 on success, otherwise negated error.
 */
static int alloc_and_dissolve_hugetlb_folio(struct folio *old_folio,
                        struct list_head *list)
{
        gfp_t gfp_mask;
        struct hstate *h;
        int nid = folio_nid(old_folio);
        struct folio *new_folio = NULL;
        int ret = 0;

retry:
        /*
         * The old_folio might have been dissolved from under our feet, so make sure
         * to carefully check the state under the lock.
         */
        spin_lock_irq(&hugetlb_lock);
        if (!folio_test_hugetlb(old_folio)) {
                /*
                 * Freed from under us. Drop new_folio too.
                 */
                goto free_new;
        } else if (folio_ref_count(old_folio)) {
                bool isolated;

                /*
                 * Someone has grabbed the folio, try to isolate it here.
                 * Fail with -EBUSY if not possible.
                 */
                spin_unlock_irq(&hugetlb_lock);
                isolated = folio_isolate_hugetlb(old_folio, list);
                ret = isolated ? 0 : -EBUSY;
                spin_lock_irq(&hugetlb_lock);
                goto free_new;
        } else if (!folio_test_hugetlb_freed(old_folio)) {
                /*
                 * Folio's refcount is 0 but it has not been enqueued in the
                 * freelist yet. Race window is small, so we can succeed here if
                 * we retry.
                 */
                spin_unlock_irq(&hugetlb_lock);
                cond_resched();
                goto retry;
        } else {
                h = folio_hstate(old_folio);
                if (!new_folio) {
                        spin_unlock_irq(&hugetlb_lock);
                        gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
                        new_folio = alloc_fresh_hugetlb_folio(h, gfp_mask,
                                                              nid, NULL);
                        if (!new_folio)
                                return -ENOMEM;
                        goto retry;
                }

                /*
                 * Ok, old_folio is still a genuine free hugepage. Remove it from
                 * the freelist and decrease the counters. These will be
                 * incremented again when calling account_new_hugetlb_folio()
                 * and enqueue_hugetlb_folio() for new_folio. The counters will
                 * remain stable since this happens under the lock.
                 */
                remove_hugetlb_folio(h, old_folio, false);

                /*
                 * Ref count on new_folio is already zero as it was dropped
                 * earlier.  It can be directly added to the pool free list.
                 */
                account_new_hugetlb_folio(h, new_folio);
                enqueue_hugetlb_folio(h, new_folio);

                /*
                 * Folio has been replaced, we can safely free the old one.
                 */
                spin_unlock_irq(&hugetlb_lock);
                update_and_free_hugetlb_folio(h, old_folio, false);
        }

        return ret;

free_new:
        spin_unlock_irq(&hugetlb_lock);
        if (new_folio)
                update_and_free_hugetlb_folio(h, new_folio, false);

        return ret;
}

int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list)
{
        int ret = -EBUSY;

        /* Not to disrupt normal path by vainly holding hugetlb_lock */
        if (!folio_test_hugetlb(folio))
                return 0;

        /*
         * Fence off gigantic pages as there is a cyclic dependency between
         * alloc_contig_range and them. Return -ENOMEM as this has the effect
         * of bailing out right away without further retrying.
         */
        if (order_is_gigantic(folio_order(folio)))
                return -ENOMEM;

        if (folio_ref_count(folio) && folio_isolate_hugetlb(folio, list))
                ret = 0;
        else if (!folio_ref_count(folio))
                ret = alloc_and_dissolve_hugetlb_folio(folio, list);

        return ret;
}

/*
 *  replace_free_hugepage_folios - Replace free hugepage folios in a given pfn
 *  range with new folios.
 *  @start_pfn: start pfn of the given pfn range
 *  @end_pfn: end pfn of the given pfn range
 *  Returns 0 on success, otherwise negated error.
 */
int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn)
{
        unsigned long nr = 0;
        struct page *page;
        struct hstate *h;
        LIST_HEAD(list);
        int ret = 0;

        /* Avoid pfn iterations if no free non-gigantic huge pages */
        for_each_hstate(h) {
                if (hstate_is_gigantic(h))
                        continue;

                nr += h->free_huge_pages;
                if (nr)
                        break;
        }

        if (!nr)
                return 0;

        while (start_pfn < end_pfn) {
                page = pfn_to_page(start_pfn);
                nr = 1;

                if (PageHuge(page) || PageCompound(page)) {
                        struct folio *folio = page_folio(page);

                        nr = folio_nr_pages(folio) - folio_page_idx(folio, page);

                        /*
                         * Don't disrupt normal path by vainly holding
                         * hugetlb_lock
                         */
                        if (folio_test_hugetlb(folio) && !folio_ref_count(folio)) {
                                if (order_is_gigantic(folio_order(folio))) {
                                        ret = -ENOMEM;
                                        break;
                                }

                                ret = alloc_and_dissolve_hugetlb_folio(folio, &list);
                                if (ret)
                                        break;

                                putback_movable_pages(&list);
                        }
                } else if (PageBuddy(page)) {
                        /*
                         * Buddy order check without zone lock is unsafe and
                         * the order is maybe invalid, but race should be
                         * small, and the worst thing is skipping free hugetlb.
                         */
                        const unsigned int order = buddy_order_unsafe(page);

                        if (order <= MAX_PAGE_ORDER)
                                nr = 1UL << order;
                }
                start_pfn += nr;
        }

        return ret;
}

void wait_for_freed_hugetlb_folios(void)
{
        if (llist_empty(&hpage_freelist))
                return;

        flush_work(&free_hpage_work);
}

/**
 * hugetlb_alloc_folio - Allocate a hugetlb folio.
 * @h: Hugetlb state control block.
 * @mpoli: Interpreted memory policy to use for allocation.
 * @alloc_flags: Flags controlling the allocation behavior.
 *
 * Allocates a hugetlb folio and handles cgroup charging and global hstate
 * reservations.
 *
 * Return: A pointer to the allocated folio, or an ERR_PTR on failure.
 *         -ENOSPC if cgroup charging fails or no folio is available.
 *         -ENOMEM if mem cgroup charging fails.
 */
struct folio *hugetlb_alloc_folio(struct hstate *h,
                struct mempolicy_interpreted *mpoli, u8 alloc_flags)
{
        bool charge_hugetlb_cgroup_rsvd = alloc_flags &
                                          HUGETLB_ALLOC_CHARG_CGROUP_RSVD;
        bool use_global_reservation = alloc_flags &
                                      HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS;
        size_t nr_pages = pages_per_huge_page(h);
        struct hugetlb_cgroup *h_cg_rsvd = NULL;
        struct hugetlb_cgroup *h_cg = NULL;
        gfp_t gfp = htlb_alloc_mask(h);
        int idx = hstate_index(h);
        struct folio *folio;
        int ret;

        if (charge_hugetlb_cgroup_rsvd &&
            hugetlb_cgroup_charge_cgroup_rsvd(idx, nr_pages, &h_cg_rsvd))
                return ERR_PTR(-ENOSPC);

        if (hugetlb_cgroup_charge_cgroup(idx, nr_pages, &h_cg)) {
                ret = -ENOSPC;
                goto err_uncharge_hugetlb_cgroup_rsvd;
        }

        spin_lock_irq(&hugetlb_lock);

        folio = NULL;
        if (use_global_reservation || available_huge_pages(h))
                folio = dequeue_hugetlb_folio(h, gfp, mpoli);

        if (!folio) {
                spin_unlock_irq(&hugetlb_lock);
                folio = alloc_buddy_hugetlb_folio(h, gfp, mpoli);
                if (!folio) {
                        ret = -ENOSPC;
                        goto err_uncharge_hugetlb_cgroup;
                }
                spin_lock_irq(&hugetlb_lock);
                list_add(&folio->lru, &h->hugepage_activelist);
                folio_ref_unfreeze(folio, 1);
        }

        if (use_global_reservation) {
                folio_set_hugetlb_restore_reserve(folio);
                h->resv_huge_pages--;
        }

        hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio);

        if (charge_hugetlb_cgroup_rsvd) {
                hugetlb_cgroup_commit_charge_rsvd(idx, nr_pages, h_cg_rsvd,
                                                  folio);
        }

        spin_unlock_irq(&hugetlb_lock);

        ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL);
        /*
         * Unconditionally increment NR_HUGETLB here because if
         * mem_cgroup_charge_hugetlb failed, freeing the page will
         * decrement NR_HUGETLB.
         */
        lruvec_stat_mod_folio(folio, NR_HUGETLB, nr_pages);

        if (ret == -ENOMEM) {
                free_huge_folio(folio);
                /*
                 * Skip uncharging hugetlb_cgroup since the charges
                 * were committed to the folio and freeing the folio
                 * would have cleared those up.
                 */
                return ERR_PTR(ret);
        }

        return folio;

 err_uncharge_hugetlb_cgroup:
        hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg);
 err_uncharge_hugetlb_cgroup_rsvd:
        if (charge_hugetlb_cgroup_rsvd)
                hugetlb_cgroup_uncharge_cgroup_rsvd(idx, nr_pages, h_cg_rsvd);

        return ERR_PTR(ret);
}

typedef enum {
        /*
         * For either 0/1: we checked the per-vma resv map, and one resv
         * count either can be reused (0), or an extra needed (1).
         */
        MAP_CHG_REUSE = 0,
        MAP_CHG_NEEDED = 1,
        /*
         * Cannot use per-vma resv count can be used, hence a new resv
         * count is enforced.
         *
         * NOTE: This is mostly identical to MAP_CHG_NEEDED, except
         * that currently vma_needs_reservation() has an unwanted side
         * effect to either use end() or commit() to complete the
         * transaction. Hence it needs to differentiate from NEEDED.
         */
        MAP_CHG_ENFORCED = 2,
} map_chg_state;

/*
 * NOTE! "cow_from_owner" represents a very hacky usage only used in CoW
 * faults of hugetlb private mappings on top of a non-page-cache folio (in
 * which case even if there's a private vma resv map it won't cover such
 * allocation).  New call sites should (probably) never set it to true!!
 * When it's set, the allocation will bypass all vma level reservations.
 */
struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
                                    unsigned long addr, bool cow_from_owner)
{
        struct hugepage_subpool *spool = subpool_vma(vma);
        struct hstate *h = hstate_vma(vma);
        struct folio *folio;
        long retval, gbl_chg, gbl_reserve;
        map_chg_state map_chg;
        struct mempolicy_interpreted mpoli;
        gfp_t gfp = htlb_alloc_mask(h);
        struct mempolicy *mpol;
        nodemask_t *nodemask;
        u8 alloc_flags = 0;
        int nid;
        int ret;

        /* Whether we need a separate per-vma reservation? */
        if (cow_from_owner) {
                /*
                 * Special case!  Since it's a CoW on top of a reserved
                 * page, the private resv map doesn't count.  So it cannot
                 * consume the per-vma resv map even if it's reserved.
                 */
                map_chg = MAP_CHG_ENFORCED;
        } else {
                /*
                 * Examine the region/reserve map to determine if the process
                 * has a reservation for the page to be allocated.  A return
                 * code of zero indicates a reservation exists (no change).
                 */
                retval = vma_needs_reservation(h, vma, addr);
                if (retval < 0)
                        return ERR_PTR(-ENOMEM);
                map_chg = retval ? MAP_CHG_NEEDED : MAP_CHG_REUSE;
        }

        /*
         * Whether we need a separate global reservation?
         *
         * Processes that did not create the mapping will have no
         * reserves as indicated by the region/reserve map. Check
         * that the allocation will not exceed the subpool limit.
         * Or if it can get one from the pool reservation directly.
         */
        if (map_chg) {
                gbl_chg = hugepage_subpool_get_pages(spool, 1);
                if (gbl_chg < 0) {
                        ret = -ENOSPC;
                        goto out_end_reservation;
                }
        } else {
                /*
                 * If we have the vma reservation ready, no need for extra
                 * global reservation.
                 */
                gbl_chg = 0;
        }

        /*
         * If allocation doesn't reuse a reservation in the resv_map,
         * charge for the reservation.
         */
        if (map_chg != MAP_CHG_REUSE)
                alloc_flags |= HUGETLB_ALLOC_CHARG_CGROUP_RSVD;

        /*
         * gbl_chg == 0 indicates a reservation exists for this
         * allocation, so try to use it.
         */
        if (gbl_chg == 0)
                alloc_flags |= HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS;

        /* Takes reference on mpol. */
        nid = huge_node(vma, addr, gfp, &mpol, &nodemask);
        mpoli = (struct mempolicy_interpreted){
                .nid = nid,
#ifdef CONFIG_NUMA
                .mode = mpol ? mpol->mode : MPOL_DEFAULT,
#else
                .mode = MPOL_DEFAULT,
#endif
                .nodemask = nodemask,
        };

        folio = hugetlb_alloc_folio(h, &mpoli, alloc_flags);

        mpol_cond_put(mpol);

        if (IS_ERR(folio)) {
                ret = PTR_ERR(folio);
                goto out_subpool_put;
        }

        hugetlb_set_folio_subpool(folio, spool);

        if (map_chg != MAP_CHG_ENFORCED) {
                /* commit() is only needed if the map_chg is not enforced */
                retval = vma_commit_reservation(h, vma, addr);
                /*
                 * Check for possible race conditions. When it happens..
                 * The page was added to the reservation map between
                 * vma_needs_reservation and vma_commit_reservation.
                 * This indicates a race with hugetlb_reserve_pages.
                 * Adjust for the subpool count incremented above AND
                 * in hugetlb_reserve_pages for the same page.        Also,
                 * the reservation count added in hugetlb_reserve_pages
                 * no longer applies.
                 */
                if (unlikely(map_chg == MAP_CHG_NEEDED && retval == 0)) {
                        long rsv_adjust;

                        rsv_adjust = hugepage_subpool_put_pages(spool, 1);
                        hugetlb_acct_memory(h, -rsv_adjust);
                        spin_lock_irq(&hugetlb_lock);
                        hugetlb_cgroup_uncharge_folio_rsvd(
                            hstate_index(h), pages_per_huge_page(h), folio);
                        spin_unlock_irq(&hugetlb_lock);
                }
        }

        return folio;

out_subpool_put:
        /*
         * put page to subpool iff the quota of subpool's rsv_hpages is used
         * during hugepage_subpool_get_pages.
         */
        if (map_chg && !gbl_chg) {
                gbl_reserve = hugepage_subpool_put_pages(spool, 1);
                hugetlb_acct_memory(h, -gbl_reserve);
        }

out_end_reservation:
        if (map_chg != MAP_CHG_ENFORCED)
                vma_end_reservation(h, vma, addr);
        return ERR_PTR(ret);
}

static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact)
{
        if (hugetlb_early_cma(h))
                return hugetlb_cma_alloc_bootmem(h, nid, node_exact);

        return memblock_alloc_hugetlb(huge_page_size(h), nid, node_exact);
}

void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid)
        __attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid)
{
        int nr_nodes, node = nid;

        /* do node specific alloc */
        if (nid != NUMA_NO_NODE)
                return alloc_bootmem(h, node, true);

        /* allocate from next node when distributing huge pages */
        for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node,
                                    &hugetlb_bootmem_nodes)
                return alloc_bootmem(h, node, false);

        return NULL;
}

static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
{
        unsigned long pfn;
        unsigned int nid_request = nid;
        struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid);

        if (!m)
                return false;

        pfn = PHYS_PFN(__pa(m));
        nid = early_pfn_to_nid(pfn);
        /*
         * Use the beginning of the huge page to store the huge_bootmem_page
         * struct (until gather_bootmem puts them into the mem_map).
         *
         * Put them into a private list first because mem_map is not up yet.
         */
        INIT_LIST_HEAD(&m->list);
        m->hstate = h;
        m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0;

        /* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */
        if (!hugetlb_early_cma(h) &&
            pfn_range_intersects_zones(nid, pfn, pages_per_huge_page(h))) {
                /*
                 * If the allocated page is on a different node than requested
                 * (e.g., on PowerPC LPARs), put it on the requested node's list,
                 * because hugetlb_free_cross_zone_pages() only frees cross-zone
                 * pages belonging to the requested node.
                 */
                if (WARN_ON_ONCE(nid_request != NUMA_NO_NODE && nid != nid_request))
                        list_add(&m->list, &huge_boot_pages[nid_request]);
                else
                        list_add(&m->list, &huge_boot_pages[nid]);
        } else {
                list_add_tail(&m->list, &huge_boot_pages[nid]);
                m->flags |= HUGE_BOOTMEM_ZONES_VALID;
                /*
                 * Only initialize the head struct page in memmap_init_reserved_pages,
                 * rest of the struct pages will be initialized by the HugeTLB
                 * subsystem itself.
                 * The head struct page is used to get folio information by the HugeTLB
                 * subsystem like zone id and node id.
                 */
                memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE),
                                huge_page_size(h) - PAGE_SIZE);
        }

        return true;
}

/* Initialize [start_page:end_page_number] tail struct pages of a hugepage */
static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
                                        struct hstate *h,
                                        unsigned long start_page_number,
                                        unsigned long end_page_number)
{
        enum zone_type zone = folio_zonenum(folio);
        int nid = folio_nid(folio);
        struct page *page = folio_page(folio, start_page_number);
        unsigned long head_pfn = folio_pfn(folio);
        unsigned long pfn, end_pfn = head_pfn + end_page_number;
        unsigned int order = huge_page_order(h);

        /*
         * As we marked all tail pages with memblock_reserved_mark_noinit(),
         * we must initialize them ourselves here.
         */
        for (pfn = head_pfn + start_page_number; pfn < end_pfn; page++, pfn++) {
                __init_single_page(page, pfn, zone, nid);
                prep_compound_tail(page, &folio->page, order);
                set_page_count(page, 0);
        }
}

static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
                                              struct hstate *h,
                                              unsigned long nr_pages)
{
        int ret;

        /*
         * This is an open-coded prep_compound_page() whereby we avoid
         * walking pages twice by initializing/preparing+freezing them in the
         * same go.
         */
        __folio_clear_reserved(folio);
        __folio_set_head(folio);
        ret = folio_ref_freeze(folio, 1);
        VM_BUG_ON(!ret);
        hugetlb_folio_init_tail_vmemmap(folio, h, 1, nr_pages);
        prep_compound_head(&folio->page, huge_page_order(h));
}

static bool __init hugetlb_bootmem_page_prehvo(struct huge_bootmem_page *m)
{
        return m->flags & HUGE_BOOTMEM_HVO;
}

static bool __init hugetlb_bootmem_page_earlycma(struct huge_bootmem_page *m)
{
        return m->flags & HUGE_BOOTMEM_CMA;
}

/*
 * memblock-allocated pageblocks might not have the migrate type set
 * if marked with the 'noinit' flag. Set it to the default (MIGRATE_MOVABLE)
 * here, or MIGRATE_CMA if this was a page allocated through an early CMA
 * reservation.
 *
 * In case of vmemmap optimized folios, the tail vmemmap pages are mapped
 * read-only, but that's ok - for sparse vmemmap this does not write to
 * the page structure.
 */
static void __init hugetlb_bootmem_init_migratetype(struct folio *folio,
                                                          struct hstate *h)
{
        unsigned long nr_pages = pages_per_huge_page(h), i;

        WARN_ON_ONCE(!pageblock_aligned(folio_pfn(folio)));

        for (i = 0; i < nr_pages; i += pageblock_nr_pages) {
                if (folio_test_hugetlb_cma(folio))
                        init_cma_pageblock(folio_page(folio, i));
                else
                        init_pageblock_migratetype(folio_page(folio, i),
                                          MIGRATE_MOVABLE, false);
        }
}

static void __init prep_and_add_bootmem_folios(struct hstate *h,
                                        struct list_head *folio_list)
{
        unsigned long flags;
        struct folio *folio, *tmp_f;

        /* Send list for bulk vmemmap optimization processing */
        hugetlb_vmemmap_optimize_bootmem_folios(h, folio_list);

        list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
                if (!folio_test_hugetlb_vmemmap_optimized(folio)) {
                        /*
                         * If HVO fails, initialize all tail struct pages
                         * We do not worry about potential long lock hold
                         * time as this is early in boot and there should
                         * be no contention.
                         */
                        hugetlb_folio_init_tail_vmemmap(folio, h,
                                        HUGETLB_VMEMMAP_RESERVE_PAGES,
                                        pages_per_huge_page(h));
                }
                hugetlb_bootmem_init_migratetype(folio, h);
                /* Subdivide locks to achieve better parallel performance */
                spin_lock_irqsave(&hugetlb_lock, flags);
                account_new_hugetlb_folio(h, folio);
                enqueue_hugetlb_folio(h, folio);
                spin_unlock_irqrestore(&hugetlb_lock, flags);
        }
}

/*
 * Put bootmem huge pages into the standard lists after mem_map is up.
 * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages.
 */
static void __init gather_bootmem_prealloc_node(unsigned long nid)
{
        LIST_HEAD(folio_list);
        struct huge_bootmem_page *m, *tm;
        struct hstate *h = NULL, *prev_h = NULL;

        list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) {
                struct page *page = virt_to_page(m);
                struct folio *folio = (void *)page;

                h = m->hstate;
                /*
                 * It is possible to have multiple huge page sizes (hstates)
                 * in this list.  If so, process each size separately.
                 */
                if (h != prev_h && prev_h != NULL)
                        prep_and_add_bootmem_folios(prev_h, &folio_list);
                prev_h = h;

                VM_BUG_ON(!hstate_is_gigantic(h));
                WARN_ON(folio_ref_count(folio) != 1);

                hugetlb_folio_init_vmemmap(folio, h,
                                           HUGETLB_VMEMMAP_RESERVE_PAGES);
                init_new_hugetlb_folio(folio);

                if (hugetlb_bootmem_page_prehvo(m))
                        /*
                         * If pre-HVO was done, just set the
                         * flag, the HVO code will then skip
                         * this folio.
                         */
                        folio_set_hugetlb_vmemmap_optimized(folio);

                if (hugetlb_bootmem_page_earlycma(m))
                        folio_set_hugetlb_cma(folio);

                list_add(&folio->lru, &folio_list);

                /*
                 * We need to restore the 'stolen' pages to totalram_pages
                 * in order to fix confusing memory reports from free(1) and
                 * other side-effects, like CommitLimit going negative.
                 *
                 * For CMA pages, this is done in init_cma_pageblock
                 * (via hugetlb_bootmem_init_migratetype), so skip it here.
                 */
                if (!folio_test_hugetlb_cma(folio))
                        adjust_managed_page_count(page, pages_per_huge_page(h));
                cond_resched();
        }

        prep_and_add_bootmem_folios(h, &folio_list);
}

static void __init gather_bootmem_prealloc_parallel(unsigned long start,
                                                    unsigned long end, void *arg)
{
        int nid;

        for (nid = start; nid < end; nid++)
                gather_bootmem_prealloc_node(nid);
}

void __init hugetlb_bootmem_struct_page_init(void)
{
        struct padata_mt_job job = {
                .thread_fn        = gather_bootmem_prealloc_parallel,
                .fn_arg                = NULL,
                .start                = 0,
                .size                = nr_node_ids,
                .align                = 1,
                .min_chunk        = 1,
                .max_threads        = num_node_state(N_MEMORY),
                .numa_aware        = true,
        };
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
        struct zone *zone;

        for_each_zone(zone) {
                for (int i = 0; i < NR_VMEMMAP_TAILS; i++) {
                        struct page *tail, *p;
                        unsigned int order;

                        tail = zone->vmemmap_tails[i];
                        if (!tail)
                                continue;

                        order = i + VMEMMAP_TAIL_MIN_ORDER;
                        p = page_to_virt(tail);
                        /*
                         * prep_and_add_bootmem_folios() can access pageblock
                         * flags on bootmem HugeTLB pages, so initialize the
                         * shared tail struct pages here before bootmem folios
                         * start using them.
                         */
                        for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++)
                                init_compound_tail(p + j, NULL, order, zone);
                }
        }
#endif

        padata_do_multithreaded(&job);
}

static unsigned long __init hugetlb_free_cross_zone_pages(struct hstate *h, int nid)
{
        unsigned long freed = 0;
        struct huge_bootmem_page *m, *tmp;

        if (!hstate_is_gigantic(h))
                return freed;

        list_for_each_entry_safe(m, tmp, &huge_boot_pages[nid], list) {
                if (m->flags & HUGE_BOOTMEM_ZONES_VALID)
                        break;

                list_del(&m->list);
                memblock_free(m, huge_page_size(h));
                freed++;
        }

        if (freed) {
                char buf[32];

                string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, sizeof(buf));
                pr_warn("HugeTLB: freed %lu cross-zone hugepages of size %s on node %d.\n",
                        freed, buf, nid);
        }

        return freed;
}

static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
{
        unsigned long i;
        char buf[32];
        LIST_HEAD(folio_list);

        for (i = 0; i < h->max_huge_pages_node[nid]; ++i) {
                if (hstate_is_gigantic(h)) {
                        if (!alloc_bootmem_huge_page(h, nid))
                                break;
                } else {
                        struct folio *folio;
                        gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;

                        folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
                                        &node_states[N_MEMORY], NULL);
                        if (!folio && !list_empty(&folio_list) &&
                            hugetlb_vmemmap_optimizable_size(h)) {
                                prep_and_add_allocated_folios(h, &folio_list);
                                INIT_LIST_HEAD(&folio_list);
                                folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
                                                &node_states[N_MEMORY], NULL);
                        }
                        if (!folio)
                                break;
                        list_add(&folio->lru, &folio_list);
                }
                cond_resched();
        }

        i -= hugetlb_free_cross_zone_pages(h, nid);

        if (!list_empty(&folio_list))
                prep_and_add_allocated_folios(h, &folio_list);

        if (i == h->max_huge_pages_node[nid])
                return;

        string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
        pr_warn("HugeTLB: allocating %u of page size %s failed node%d.  Only allocated %lu hugepages.\n",
                h->max_huge_pages_node[nid], buf, nid, i);
        h->max_huge_pages -= (h->max_huge_pages_node[nid] - i);
        h->max_huge_pages_node[nid] = i;
}

static bool __init hugetlb_hstate_alloc_pages_specific_nodes(struct hstate *h)
{
        int i;
        bool node_specific_alloc = false;

        for_each_online_node(i) {
                if (h->max_huge_pages_node[i] > 0) {
                        hugetlb_hstate_alloc_pages_onenode(h, i);
                        node_specific_alloc = true;
                }
        }

        return node_specific_alloc;
}

static void __init hugetlb_hstate_alloc_pages_errcheck(unsigned long allocated, struct hstate *h)
{
        if (allocated < h->max_huge_pages) {
                char buf[32];

                string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
                pr_warn("HugeTLB: allocating %lu of page size %s failed.  Only allocated %lu hugepages.\n",
                        h->max_huge_pages, buf, allocated);
                h->max_huge_pages = allocated;
        }
}

static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned long end, void *arg)
{
        struct hstate *h = (struct hstate *)arg;
        int i, num = end - start;
        nodemask_t node_alloc_noretry;
        LIST_HEAD(folio_list);
        int next_node = first_online_node;

        /* Bit mask controlling how hard we retry per-node allocations.*/
        nodes_clear(node_alloc_noretry);

        for (i = 0; i < num; ++i) {
                struct folio *folio;

                if (hugetlb_vmemmap_optimizable_size(h) &&
                    (si_mem_available() == 0) && !list_empty(&folio_list)) {
                        prep_and_add_allocated_folios(h, &folio_list);
                        INIT_LIST_HEAD(&folio_list);
                }
                folio = alloc_pool_huge_folio(h, &node_states[N_MEMORY],
                                                &node_alloc_noretry, &next_node);
                if (!folio)
                        break;

                list_move(&folio->lru, &folio_list);
                cond_resched();
        }

        prep_and_add_allocated_folios(h, &folio_list);
}

static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h)
{
        int nid;
        unsigned long i;

        for (i = 0; i < h->max_huge_pages; ++i) {
                if (!alloc_bootmem_huge_page(h, NUMA_NO_NODE))
                        break;
                cond_resched();
        }

        for_each_node(nid)
                i -= hugetlb_free_cross_zone_pages(h, nid);

        return i;
}

static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h)
{
        struct padata_mt_job job = {
                .fn_arg                = h,
                .align                = 1,
                .numa_aware        = true
        };

        unsigned long jiffies_start;
        unsigned long jiffies_end;
        unsigned long remaining;

        job.thread_fn        = hugetlb_pages_alloc_boot_node;

        /*
         * job.max_threads is 25% of the available cpu threads by default.
         *
         * On large servers with terabytes of memory, huge page allocation
         * can consume a considerably amount of time.
         *
         * Tests below show how long it takes to allocate 1 TiB of memory with 2MiB huge pages.
         * 2MiB huge pages. Using more threads can significantly improve allocation time.
         *
         * +-----------------------+-------+-------+-------+-------+-------+
         * | threads               |   8   |   16  |   32  |   64  |   128 |
         * +-----------------------+-------+-------+-------+-------+-------+
         * | skylake      144 cpus |   44s |   22s |   16s |   19s |   20s |
         * | cascade lake 192 cpus |   39s |   20s |   11s |   10s |    9s |
         * +-----------------------+-------+-------+-------+-------+-------+
         */
        if (hugepage_allocation_threads == 0) {
                hugepage_allocation_threads = num_online_cpus() / 4;
                hugepage_allocation_threads = max(hugepage_allocation_threads, 1);
        }

        job.max_threads        = hugepage_allocation_threads;

        jiffies_start = jiffies;
        do {
                remaining = h->max_huge_pages - h->nr_huge_pages;

                job.start     = h->nr_huge_pages;
                job.size      = remaining;
                job.min_chunk = remaining / hugepage_allocation_threads;
                padata_do_multithreaded(&job);

                if (h->nr_huge_pages == h->max_huge_pages)
                        break;

                /*
                 * Retry only if the vmemmap optimization might have been able to free
                 * some memory back to the system.
                 */
                if (!hugetlb_vmemmap_optimizable(h))
                        break;

                /* Continue if progress was made in last iteration */
        } while (remaining != (h->max_huge_pages - h->nr_huge_pages));

        jiffies_end = jiffies;

        pr_info("HugeTLB: allocation took %dms with hugepage_allocation_threads=%ld\n",
                jiffies_to_msecs(jiffies_end - jiffies_start),
                hugepage_allocation_threads);

        return h->nr_huge_pages;
}

/*
 * NOTE: this routine is called in different contexts for gigantic and
 * non-gigantic pages.
 * - For gigantic pages, this is called early in the boot process and
 *   pages are allocated from memblock allocated or something similar.
 *   Gigantic pages are actually added to pools later with the routine
 *   hugetlb_bootmem_struct_page_init.
 * - For non-gigantic pages, this is called later in the boot process after
 *   all of mm is up and functional.  Pages are allocated from buddy and
 *   then added to hugetlb pools.
 */
static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
{
        unsigned long allocated;

        /*
         * Skip gigantic hugepages allocation if early CMA
         * reservations are not available.
         */
        if (hstate_is_gigantic(h) && hugetlb_cma_total_size() &&
            !hugetlb_early_cma(h)) {
                pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
                return;
        }

        if (!h->max_huge_pages)
                return;

        /* do node specific alloc */
        if (hugetlb_hstate_alloc_pages_specific_nodes(h))
                return;

        /* below will do all node balanced alloc */
        if (hstate_is_gigantic(h))
                allocated = hugetlb_gigantic_pages_alloc_boot(h);
        else
                allocated = hugetlb_pages_alloc_boot(h);

        hugetlb_hstate_alloc_pages_errcheck(allocated, h);
}

static void __init hugetlb_init_hstates(void)
{
        struct hstate *h, *h2;

        for_each_hstate(h) {
                /*
                 * Always reset to first_memory_node here, even if
                 * next_nid_to_alloc was set before - we can't
                 * reference hugetlb_bootmem_nodes after init, and
                 * first_memory_node is right for all further allocations.
                 */
                h->next_nid_to_alloc = first_memory_node;
                h->next_nid_to_free = first_memory_node;

                /* oversize hugepages were init'ed in early boot */
                if (!hstate_is_gigantic(h))
                        hugetlb_hstate_alloc_pages(h);

                /*
                 * Set demote order for each hstate.  Note that
                 * h->demote_order is initially 0.
                 * - We can not demote gigantic pages if runtime freeing
                 *   is not supported, so skip this.
                 * - If CMA allocation is possible, we can not demote
                 *   HUGETLB_PAGE_ORDER or smaller size pages.
                 */
                if (hstate_is_gigantic_no_runtime(h))
                        continue;
                if (hugetlb_cma_total_size() && h->order <= HUGETLB_PAGE_ORDER)
                        continue;
                for_each_hstate(h2) {
                        if (h2 == h)
                                continue;
                        if (h2->order < h->order &&
                            h2->order > h->demote_order)
                                h->demote_order = h2->order;
                }
        }
}

static void __init report_hugepages(void)
{
        struct hstate *h;

        for_each_hstate(h) {
                char buf[32];

                string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
                pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n",
                        buf, h->nr_huge_pages);
                pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n",
                        hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf);
        }
}

#ifdef CONFIG_HIGHMEM
static void try_to_free_low(struct hstate *h, unsigned long count,
                                                nodemask_t *nodes_allowed)
{
        int i;
        LIST_HEAD(page_list);

        lockdep_assert_held(&hugetlb_lock);
        if (hstate_is_gigantic(h))
                return;

        /*
         * Collect pages to be freed on a list, and free after dropping lock
         */
        for_each_node_mask(i, *nodes_allowed) {
                struct folio *folio, *next;
                struct list_head *freel = &h->hugepage_freelists[i];
                list_for_each_entry_safe(folio, next, freel, lru) {
                        if (count >= h->nr_huge_pages)
                                goto out;
                        if (folio_test_highmem(folio))
                                continue;
                        remove_hugetlb_folio(h, folio, false);
                        list_add(&folio->lru, &page_list);
                }
        }

out:
        spin_unlock_irq(&hugetlb_lock);
        update_and_free_pages_bulk(h, &page_list);
        spin_lock_irq(&hugetlb_lock);
}
#else
static inline void try_to_free_low(struct hstate *h, unsigned long count,
                                                nodemask_t *nodes_allowed)
{
}
#endif

/*
 * Increment or decrement surplus_huge_pages.  Keep node-specific counters
 * balanced by operating on them in a round-robin fashion.
 * Returns 1 if an adjustment was made.
 */
static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
                                int delta)
{
        int nr_nodes, node;

        lockdep_assert_held(&hugetlb_lock);
        VM_BUG_ON(delta != -1 && delta != 1);

        if (delta < 0) {
                for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, nodes_allowed) {
                        if (h->surplus_huge_pages_node[node])
                                goto found;
                }
        } else {
                for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
                        if (h->surplus_huge_pages_node[node] <
                                        h->nr_huge_pages_node[node])
                                goto found;
                }
        }
        return 0;

found:
        h->surplus_huge_pages += delta;
        h->surplus_huge_pages_node[node] += delta;
        return 1;
}

#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
                              nodemask_t *nodes_allowed)
{
        unsigned long persistent_free_count;
        unsigned long min_count;
        unsigned long allocated;
        struct folio *folio;
        LIST_HEAD(page_list);
        NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);

        /*
         * Bit mask controlling how hard we retry per-node allocations.
         * If we can not allocate the bit mask, do not attempt to allocate
         * the requested huge pages.
         */
        if (node_alloc_noretry)
                nodes_clear(*node_alloc_noretry);
        else
                return -ENOMEM;

        /*
         * resize_lock mutex prevents concurrent adjustments to number of
         * pages in hstate via the proc/sysfs interfaces.
         */
        mutex_lock(&h->resize_lock);
        flush_free_hpage_work(h);
        spin_lock_irq(&hugetlb_lock);

        /*
         * Check for a node specific request.
         * Changing node specific huge page count may require a corresponding
         * change to the global count.  In any case, the passed node mask
         * (nodes_allowed) will restrict alloc/free to the specified node.
         */
        if (nid != NUMA_NO_NODE) {
                unsigned long old_count = count;

                count += persistent_huge_pages(h) -
                         (h->nr_huge_pages_node[nid] -
                          h->surplus_huge_pages_node[nid]);
                /*
                 * User may have specified a large count value which caused the
                 * above calculation to overflow.  In this case, they wanted
                 * to allocate as many huge pages as possible.  Set count to
                 * largest possible value to align with their intention.
                 */
                if (count < old_count)
                        count = ULONG_MAX;
        }

        /*
         * Gigantic pages runtime allocation depend on the capability for large
         * page range allocation.
         * If the system does not provide this feature, return an error when
         * the user tries to allocate gigantic pages but let the user free the
         * boottime allocated gigantic pages.
         */
        if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
                if (count > persistent_huge_pages(h)) {
                        spin_unlock_irq(&hugetlb_lock);
                        mutex_unlock(&h->resize_lock);
                        NODEMASK_FREE(node_alloc_noretry);
                        return -EINVAL;
                }
                /* Fall through to decrease pool */
        }

        /*
         * Increase the pool size
         * First take pages out of surplus state.  Then make up the
         * remaining difference by allocating fresh huge pages.
         *
         * We might race with alloc_surplus_hugetlb_folio() here and be unable
         * to convert a surplus huge page to a normal huge page. That is
         * not critical, though, it just means the overall size of the
         * pool might be one hugepage larger than it needs to be, but
         * within all the constraints specified by the sysctls.
         */
        while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
                if (!adjust_pool_surplus(h, nodes_allowed, -1))
                        break;
        }

        allocated = 0;
        while (count > (persistent_huge_pages(h) + allocated)) {
                /*
                 * If this allocation races such that we no longer need the
                 * page, free_huge_folio will handle it by freeing the page
                 * and reducing the surplus.
                 */
                spin_unlock_irq(&hugetlb_lock);

                /* yield cpu to avoid soft lockup */
                cond_resched();

                folio = alloc_pool_huge_folio(h, nodes_allowed,
                                                node_alloc_noretry,
                                                &h->next_nid_to_alloc);
                if (!folio) {
                        prep_and_add_allocated_folios(h, &page_list);
                        spin_lock_irq(&hugetlb_lock);
                        goto out;
                }

                list_add(&folio->lru, &page_list);
                allocated++;

                /* Bail for signals. Probably ctrl-c from user */
                if (signal_pending(current)) {
                        prep_and_add_allocated_folios(h, &page_list);
                        spin_lock_irq(&hugetlb_lock);
                        goto out;
                }

                spin_lock_irq(&hugetlb_lock);
        }

        /* Add allocated pages to the pool */
        if (!list_empty(&page_list)) {
                spin_unlock_irq(&hugetlb_lock);
                prep_and_add_allocated_folios(h, &page_list);
                spin_lock_irq(&hugetlb_lock);
        }

        /*
         * Decrease the pool size
         * First return free pages to the buddy allocator (being careful
         * to keep enough around to satisfy reservations).  Then place
         * pages into surplus state as needed so the pool will shrink
         * to the desired size as pages become free.
         *
         * By placing pages into the surplus state independent of the
         * overcommit value, we are allowing the surplus pool size to
         * exceed overcommit. There are few sane options here. Since
         * alloc_surplus_hugetlb_folio() is checking the global counter,
         * though, we'll note that we're not allowed to exceed surplus
         * and won't grow the pool anywhere else. Not until one of the
         * sysctls are changed, or the surplus pages go out of use.
         *
         * min_count is the expected number of persistent pages, we
         * shouldn't calculate min_count by using
         * resv_huge_pages + persistent_huge_pages() - free_huge_pages,
         * because there may exist free surplus huge pages, and this will
         * lead to subtracting twice. Free surplus huge pages come from HVO
         * failing to restore vmemmap, see comments in the callers of
         * hugetlb_vmemmap_restore_folio(). Thus, we should calculate
         * persistent free count first.
         */
        persistent_free_count = h->free_huge_pages;
        if (h->free_huge_pages > persistent_huge_pages(h)) {
                if (h->free_huge_pages > h->surplus_huge_pages)
                        persistent_free_count -= h->surplus_huge_pages;
                else
                        persistent_free_count = 0;
        }
        min_count = h->resv_huge_pages + persistent_huge_pages(h) - persistent_free_count;
        min_count = max(count, min_count);
        try_to_free_low(h, min_count, nodes_allowed);

        /*
         * Collect pages to be removed on list without dropping lock
         */
        while (min_count < persistent_huge_pages(h)) {
                folio = remove_pool_hugetlb_folio(h, nodes_allowed, 0);
                if (!folio)
                        break;

                list_add(&folio->lru, &page_list);
        }
        /* free the pages after dropping lock */
        spin_unlock_irq(&hugetlb_lock);
        update_and_free_pages_bulk(h, &page_list);
        flush_free_hpage_work(h);
        spin_lock_irq(&hugetlb_lock);

        while (count < persistent_huge_pages(h)) {
                if (!adjust_pool_surplus(h, nodes_allowed, 1))
                        break;
        }
out:
        h->max_huge_pages = persistent_huge_pages(h);
        spin_unlock_irq(&hugetlb_lock);
        mutex_unlock(&h->resize_lock);

        NODEMASK_FREE(node_alloc_noretry);

        return 0;
}

static long demote_free_hugetlb_folios(struct hstate *src, struct hstate *dst,
                                       struct list_head *src_list)
{
        long rc;
        struct folio *folio, *next;
        LIST_HEAD(dst_list);
        LIST_HEAD(ret_list);

        rc = hugetlb_vmemmap_restore_folios(src, src_list, &ret_list);
        list_splice_init(&ret_list, src_list);

        /*
         * Taking target hstate mutex synchronizes with set_max_huge_pages.
         * Without the mutex, pages added to target hstate could be marked
         * as surplus.
         *
         * Note that we already hold src->resize_lock.  To prevent deadlock,
         * use the convention of always taking larger size hstate mutex first.
         */
        mutex_lock(&dst->resize_lock);

        list_for_each_entry_safe(folio, next, src_list, lru) {
                int i;
                bool cma;

                if (folio_test_hugetlb_vmemmap_optimized(folio))
                        continue;

                cma = folio_test_hugetlb_cma(folio);

                list_del(&folio->lru);

                split_page_owner(&folio->page, huge_page_order(src), huge_page_order(dst));
                pgalloc_tag_split(folio, huge_page_order(src), huge_page_order(dst));

                for (i = 0; i < pages_per_huge_page(src); i += pages_per_huge_page(dst)) {
                        struct page *page = folio_page(folio, i);
                        /* Careful: see __split_huge_page_tail() */
                        struct folio *new_folio = (struct folio *)page;

                        clear_compound_head(page);
                        prep_compound_page(page, dst->order);

                        new_folio->mapping = NULL;
                        init_new_hugetlb_folio(new_folio);
                        /* Copy the CMA flag so that it is freed correctly */
                        if (cma)
                                folio_set_hugetlb_cma(new_folio);
                        list_add(&new_folio->lru, &dst_list);
                }
        }

        prep_and_add_allocated_folios(dst, &dst_list);

        mutex_unlock(&dst->resize_lock);

        return rc;
}

long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed,
                           unsigned long nr_to_demote)
        __must_hold(&hugetlb_lock)
{
        int nr_nodes, node;
        struct hstate *dst;
        long rc = 0;
        long nr_demoted = 0;

        lockdep_assert_held(&hugetlb_lock);

        /* We should never get here if no demote order */
        if (!src->demote_order) {
                pr_warn("HugeTLB: NULL demote order passed to demote_pool_huge_page.\n");
                return -EINVAL;                /* internal error */
        }
        dst = size_to_hstate(PAGE_SIZE << src->demote_order);

        for_each_node_mask_to_free(src, nr_nodes, node, nodes_allowed) {
                LIST_HEAD(list);
                struct folio *folio, *next;

                list_for_each_entry_safe(folio, next, &src->hugepage_freelists[node], lru) {
                        if (folio_test_hwpoison(folio))
                                continue;

                        remove_hugetlb_folio(src, folio, false);
                        list_add(&folio->lru, &list);

                        if (++nr_demoted == nr_to_demote)
                                break;
                }

                spin_unlock_irq(&hugetlb_lock);

                rc = demote_free_hugetlb_folios(src, dst, &list);

                spin_lock_irq(&hugetlb_lock);

                list_for_each_entry_safe(folio, next, &list, lru) {
                        list_del(&folio->lru);
                        add_hugetlb_folio(src, folio, false);

                        nr_demoted--;
                }

                if (rc < 0 || nr_demoted == nr_to_demote)
                        break;
        }

        /*
         * Not absolutely necessary, but for consistency update max_huge_pages
         * based on pool changes for the demoted page.
         */
        src->max_huge_pages -= nr_demoted;
        dst->max_huge_pages += nr_demoted << (huge_page_order(src) - huge_page_order(dst));

        if (rc < 0)
                return rc;

        if (nr_demoted)
                return nr_demoted;
        /*
         * Only way to get here is if all pages on free lists are poisoned.
         * Return -EBUSY so that caller will not retry.
         */
        return -EBUSY;
}

ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
                                           struct hstate *h, int nid,
                                           unsigned long count, size_t len)
{
        int err;
        nodemask_t nodes_allowed, *n_mask;

        if (hstate_is_gigantic_no_runtime(h))
                return -EINVAL;

        if (nid == NUMA_NO_NODE) {
                /*
                 * global hstate attribute
                 */
                if (!(obey_mempolicy &&
                                init_nodemask_of_mempolicy(&nodes_allowed)))
                        n_mask = &node_states[N_MEMORY];
                else
                        n_mask = &nodes_allowed;
        } else {
                /*
                 * Node specific request.  count adjustment happens in
                 * set_max_huge_pages() after acquiring hugetlb_lock.
                 */
                init_nodemask_of_node(&nodes_allowed, nid);
                n_mask = &nodes_allowed;
        }

        err = set_max_huge_pages(h, count, nid, n_mask);

        return err ? err : len;
}

static int __init hugetlb_init(void)
{
        int i;

        BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE <
                        __NR_HPAGEFLAGS);
        BUILD_BUG_ON_INVALID(HUGETLB_PAGE_ORDER > MAX_FOLIO_ORDER);

        if (!hugepages_supported()) {
                if (hugetlb_max_hstate || default_hstate_max_huge_pages)
                        pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n");
                return 0;
        }

        /*
         * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists.  Some
         * architectures depend on setup being done here.
         */
        hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
        if (!parsed_default_hugepagesz) {
                /*
                 * If we did not parse a default huge page size, set
                 * default_hstate_idx to HPAGE_SIZE hstate. And, if the
                 * number of huge pages for this default size was implicitly
                 * specified, set that here as well.
                 * Note that the implicit setting will overwrite an explicit
                 * setting.  A warning will be printed in this case.
                 */
                default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE));
                if (default_hstate_max_huge_pages) {
                        if (default_hstate.max_huge_pages) {
                                char buf[32];

                                string_get_size(huge_page_size(&default_hstate),
                                        1, STRING_UNITS_2, buf, 32);
                                pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n",
                                        default_hstate.max_huge_pages, buf);
                                pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n",
                                        default_hstate_max_huge_pages);
                        }
                        default_hstate.max_huge_pages =
                                default_hstate_max_huge_pages;

                        for_each_online_node(i)
                                default_hstate.max_huge_pages_node[i] =
                                        default_hugepages_in_node[i];
                }
        }

        hugetlb_init_hstates();
        report_hugepages();

        hugetlb_sysfs_init();
        hugetlb_cgroup_file_init();
        hugetlb_sysctl_init();

#ifdef CONFIG_SMP
        num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
#else
        num_fault_mutexes = 1;
#endif
        hugetlb_fault_mutex_table =
                kmalloc_objs(struct mutex, num_fault_mutexes);
        BUG_ON(!hugetlb_fault_mutex_table);

        for (i = 0; i < num_fault_mutexes; i++)
                mutex_init(&hugetlb_fault_mutex_table[i]);
        return 0;
}
subsys_initcall(hugetlb_init);

/* Overwritten by architectures with more huge page sizes */
bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
{
        return size == HPAGE_SIZE;
}

void __init hugetlb_add_hstate(unsigned int order)
{
        struct hstate *h;
        unsigned long i;

        if (size_to_hstate(PAGE_SIZE << order)) {
                return;
        }
        BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
        BUG_ON(order < order_base_2(__NR_USED_SUBPAGE));
        WARN_ON(order > MAX_FOLIO_ORDER);
        h = &hstates[hugetlb_max_hstate++];
        __mutex_init(&h->resize_lock, "resize mutex", &h->resize_key);
        h->order = order;
        h->mask = ~(huge_page_size(h) - 1);
        for (i = 0; i < MAX_NUMNODES; ++i)
                INIT_LIST_HEAD(&h->hugepage_freelists[i]);
        INIT_LIST_HEAD(&h->hugepage_activelist);
        snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
                                        huge_page_size(h)/SZ_1K);

        parsed_hstate = h;
}

bool __init __weak hugetlb_node_alloc_supported(void)
{
        return true;
}

static void __init hugepages_clear_pages_in_node(void)
{
        if (!hugetlb_max_hstate) {
                default_hstate_max_huge_pages = 0;
                memset(default_hugepages_in_node, 0,
                        sizeof(default_hugepages_in_node));
        } else {
                parsed_hstate->max_huge_pages = 0;
                memset(parsed_hstate->max_huge_pages_node, 0,
                        sizeof(parsed_hstate->max_huge_pages_node));
        }
}

static __init int hugetlb_add_param(char *s, int (*setup)(char *))
{
        size_t len;
        char *p;

        if (!s)
                return -EINVAL;

        if (hugetlb_param_index >= HUGE_MAX_CMDLINE_ARGS)
                return -EINVAL;

        len = strlen(s) + 1;
        if (len + hstate_cmdline_index > sizeof(hstate_cmdline_buf))
                return -EINVAL;

        p = &hstate_cmdline_buf[hstate_cmdline_index];
        memcpy(p, s, len);
        hstate_cmdline_index += len;

        hugetlb_params[hugetlb_param_index].val = p;
        hugetlb_params[hugetlb_param_index].setup = setup;

        hugetlb_param_index++;

        return 0;
}

static __init void hugetlb_parse_params(void)
{
        int i;
        struct hugetlb_cmdline *hcp;

        for (i = 0; i < hugetlb_param_index; i++) {
                hcp = &hugetlb_params[i];

                hcp->setup(hcp->val);
        }

        hugetlb_cma_validate_params();
}

/*
 * hugepages command line processing
 * hugepages normally follows a valid hugepagsz or default_hugepagsz
 * specification.  If not, ignore the hugepages value.  hugepages can also
 * be the first huge page command line  option in which case it implicitly
 * specifies the number of huge pages for the default size.
 */
static int __init hugepages_setup(char *s)
{
        unsigned long *mhp;
        static unsigned long *last_mhp;
        int node = NUMA_NO_NODE;
        int count;
        unsigned long tmp;
        char *p = s;

        if (!hugepages_supported()) {
                pr_warn("HugeTLB: hugepages unsupported, ignoring hugepages=%s cmdline\n", s);
                return 0;
        }

        if (!parsed_valid_hugepagesz) {
                pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s);
                parsed_valid_hugepagesz = true;
                return -EINVAL;
        }

        /*
         * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter
         * yet, so this hugepages= parameter goes to the "default hstate".
         * Otherwise, it goes with the previously parsed hugepagesz or
         * default_hugepagesz.
         */
        else if (!hugetlb_max_hstate)
                mhp = &default_hstate_max_huge_pages;
        else
                mhp = &parsed_hstate->max_huge_pages;

        if (mhp == last_mhp) {
                pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s);
                return 1;
        }

        while (*p) {
                count = 0;
                if (sscanf(p, "%lu%n", &tmp, &count) != 1)
                        goto invalid;
                /* Parameter is node format */
                if (p[count] == ':') {
                        if (!hugetlb_node_alloc_supported()) {
                                pr_warn("HugeTLB: architecture can't support node specific alloc, ignoring!\n");
                                return 1;
                        }
                        if (tmp >= MAX_NUMNODES || !node_online(tmp))
                                goto invalid;
                        node = array_index_nospec(tmp, MAX_NUMNODES);
                        p += count + 1;
                        /* Parse hugepages */
                        if (sscanf(p, "%lu%n", &tmp, &count) != 1)
                                goto invalid;
                        if (!hugetlb_max_hstate)
                                default_hugepages_in_node[node] = tmp;
                        else
                                parsed_hstate->max_huge_pages_node[node] = tmp;
                        *mhp += tmp;
                        /* Go to parse next node*/
                        if (p[count] == ',')
                                p += count + 1;
                        else
                                break;
                } else {
                        if (p != s)
                                goto invalid;
                        *mhp = tmp;
                        break;
                }
        }

        last_mhp = mhp;

        return 0;

invalid:
        pr_warn("HugeTLB: Invalid hugepages parameter %s\n", p);
        hugepages_clear_pages_in_node();
        return -EINVAL;
}
hugetlb_early_param("hugepages", hugepages_setup);

/*
 * hugepagesz command line processing
 * A specific huge page size can only be specified once with hugepagesz.
 * hugepagesz is followed by hugepages on the command line.  The global
 * variable 'parsed_valid_hugepagesz' is used to determine if prior
 * hugepagesz argument was valid.
 */
static int __init hugepagesz_setup(char *s)
{
        unsigned long size;
        struct hstate *h;

        if (!hugepages_supported()) {
                pr_warn("HugeTLB: hugepages unsupported, ignoring hugepagesz=%s cmdline\n", s);
                return 0;
        }

        parsed_valid_hugepagesz = false;
        size = (unsigned long)memparse(s, NULL);

        if (!arch_hugetlb_valid_size(size)) {
                pr_err("HugeTLB: unsupported hugepagesz=%s\n", s);
                return -EINVAL;
        }

        h = size_to_hstate(size);
        if (h) {
                /*
                 * hstate for this size already exists.  This is normally
                 * an error, but is allowed if the existing hstate is the
                 * default hstate.  More specifically, it is only allowed if
                 * the number of huge pages for the default hstate was not
                 * previously specified.
                 */
                if (!parsed_default_hugepagesz ||  h != &default_hstate ||
                    default_hstate.max_huge_pages) {
                        pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s);
                        return -EINVAL;
                }

                /*
                 * No need to call hugetlb_add_hstate() as hstate already
                 * exists.  But, do set parsed_hstate so that a following
                 * hugepages= parameter will be applied to this hstate.
                 */
                parsed_hstate = h;
                parsed_valid_hugepagesz = true;
                return 0;
        }

        hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
        parsed_valid_hugepagesz = true;
        return 0;
}
hugetlb_early_param("hugepagesz", hugepagesz_setup);

/*
 * default_hugepagesz command line input
 * Only one instance of default_hugepagesz allowed on command line.
 */
static int __init default_hugepagesz_setup(char *s)
{
        unsigned long size;
        int i;

        if (!hugepages_supported()) {
                pr_warn("HugeTLB: hugepages unsupported, ignoring default_hugepagesz=%s cmdline\n",
                        s);
                return 0;
        }

        parsed_valid_hugepagesz = false;
        if (parsed_default_hugepagesz) {
                pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s);
                return -EINVAL;
        }

        size = (unsigned long)memparse(s, NULL);

        if (!arch_hugetlb_valid_size(size)) {
                pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s);
                return -EINVAL;
        }

        hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
        parsed_valid_hugepagesz = true;
        parsed_default_hugepagesz = true;
        default_hstate_idx = hstate_index(size_to_hstate(size));

        /*
         * The number of default huge pages (for this size) could have been
         * specified as the first hugetlb parameter: hugepages=X.  If so,
         * then default_hstate_max_huge_pages is set.  If the default huge
         * page size is gigantic (> MAX_PAGE_ORDER), then the pages must be
         * allocated here from bootmem allocator.
         */
        if (default_hstate_max_huge_pages) {
                default_hstate.max_huge_pages = default_hstate_max_huge_pages;
                /*
                 * Since this is an early parameter, we can't check
                 * NUMA node state yet, so loop through MAX_NUMNODES.
                 */
                for (i = 0; i < MAX_NUMNODES; i++) {
                        if (default_hugepages_in_node[i] != 0)
                                default_hstate.max_huge_pages_node[i] =
                                        default_hugepages_in_node[i];
                }
                default_hstate_max_huge_pages = 0;
        }

        return 0;
}
hugetlb_early_param("default_hugepagesz", default_hugepagesz_setup);

void __init hugetlb_bootmem_set_nodes(void)
{
        int i, nid;

        if (!nodes_empty(hugetlb_bootmem_nodes))
                return;

        for_each_mem_pfn_range(i, MAX_NUMNODES, NULL, NULL, &nid)
                node_set(nid, hugetlb_bootmem_nodes);
}

void __init hugetlb_bootmem_alloc(void)
{
        struct hstate *h;
        int i;

        hugetlb_bootmem_set_nodes();

        for (i = 0; i < MAX_NUMNODES; i++)
                INIT_LIST_HEAD(&huge_boot_pages[i]);

        hugetlb_parse_params();

        for_each_hstate(h) {
                h->next_nid_to_alloc = first_online_node;

                if (hstate_is_gigantic(h))
                        hugetlb_hstate_alloc_pages(h);
        }
}

/*
 * hugepage_alloc_threads command line parsing.
 *
 * When set, use this specific number of threads for the boot
 * allocation of hugepages.
 */
static int __init hugepage_alloc_threads_setup(char *s)
{
        unsigned long allocation_threads;

        if (kstrtoul(s, 0, &allocation_threads) != 0)
                return 1;

        if (allocation_threads == 0)
                return 1;

        hugepage_allocation_threads = allocation_threads;

        return 1;
}
__setup("hugepage_alloc_threads=", hugepage_alloc_threads_setup);

static unsigned int allowed_mems_nr(struct hstate *h)
{
        int node;
        unsigned int nr = 0;
        nodemask_t *mbind_nodemask;
        unsigned int *array = h->free_huge_pages_node;
        gfp_t gfp_mask = htlb_alloc_mask(h);

        mbind_nodemask = policy_mbind_nodemask(gfp_mask);
        for_each_node_mask(node, cpuset_current_mems_allowed) {
                if (!mbind_nodemask || node_isset(node, *mbind_nodemask))
                        nr += array[node];
        }

        return nr;
}

void hugetlb_report_meminfo(struct seq_file *m)
{
        struct hstate *h;
        unsigned long total = 0;

        if (!hugepages_supported())
                return;

        for_each_hstate(h) {
                unsigned long count = h->nr_huge_pages;

                total += huge_page_size(h) * count;

                if (h == &default_hstate)
                        seq_printf(m,
                                   "HugePages_Total:   %5lu\n"
                                   "HugePages_Free:    %5lu\n"
                                   "HugePages_Rsvd:    %5lu\n"
                                   "HugePages_Surp:    %5lu\n"
                                   "Hugepagesize:   %8lu kB\n",
                                   count,
                                   h->free_huge_pages,
                                   h->resv_huge_pages,
                                   h->surplus_huge_pages,
                                   huge_page_size(h) / SZ_1K);
        }

        seq_printf(m, "Hugetlb:        %8lu kB\n", total / SZ_1K);
}

int hugetlb_report_node_meminfo(char *buf, int len, int nid)
{
        struct hstate *h = &default_hstate;

        if (!hugepages_supported())
                return 0;

        return sysfs_emit_at(buf, len,
                             "Node %d HugePages_Total: %5u\n"
                             "Node %d HugePages_Free:  %5u\n"
                             "Node %d HugePages_Surp:  %5u\n",
                             nid, h->nr_huge_pages_node[nid],
                             nid, h->free_huge_pages_node[nid],
                             nid, h->surplus_huge_pages_node[nid]);
}

void hugetlb_show_meminfo_node(int nid)
{
        struct hstate *h;

        if (!hugepages_supported())
                return;

        for_each_hstate(h)
                printk("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
                        nid,
                        h->nr_huge_pages_node[nid],
                        h->free_huge_pages_node[nid],
                        h->surplus_huge_pages_node[nid],
                        huge_page_size(h) / SZ_1K);
}

void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
{
        seq_printf(m, "HugetlbPages:\t%8lu kB\n",
                   K(atomic_long_read(&mm->hugetlb_usage)));
}

/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
unsigned long hugetlb_total_pages(void)
{
        struct hstate *h;
        unsigned long nr_total_pages = 0;

        for_each_hstate(h)
                nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
        return nr_total_pages;
}

static int hugetlb_acct_memory(struct hstate *h, long delta)
{
        int ret = -ENOMEM;

        if (!delta)
                return 0;

        spin_lock_irq(&hugetlb_lock);
        /*
         * When cpuset is configured, it breaks the strict hugetlb page
         * reservation as the accounting is done on a global variable. Such
         * reservation is completely rubbish in the presence of cpuset because
         * the reservation is not checked against page availability for the
         * current cpuset. Application can still potentially OOM'ed by kernel
         * with lack of free htlb page in cpuset that the task is in.
         * Attempt to enforce strict accounting with cpuset is almost
         * impossible (or too ugly) because cpuset is too fluid that
         * task or memory node can be dynamically moved between cpusets.
         *
         * The change of semantics for shared hugetlb mapping with cpuset is
         * undesirable. However, in order to preserve some of the semantics,
         * we fall back to check against current free page availability as
         * a best attempt and hopefully to minimize the impact of changing
         * semantics that cpuset has.
         *
         * Apart from cpuset, we also have memory policy mechanism that
         * also determines from which node the kernel will allocate memory
         * in a NUMA system. So similar to cpuset, we also should consider
         * the memory policy of the current task. Similar to the description
         * above.
         */
        if (delta > 0) {
                if (gather_surplus_pages(h, delta) < 0)
                        goto out;

                if (delta > allowed_mems_nr(h)) {
                        return_unused_surplus_pages(h, delta);
                        goto out;
                }
        }

        ret = 0;
        if (delta < 0)
                return_unused_surplus_pages(h, (unsigned long) -delta);

out:
        spin_unlock_irq(&hugetlb_lock);
        return ret;
}

static void hugetlb_vm_op_open(struct vm_area_struct *vma)
{
        struct resv_map *resv = vma_resv_map(vma);

        /*
         * HPAGE_RESV_OWNER indicates a private mapping.
         * This new VMA should share its siblings reservation map if present.
         * The VMA will only ever have a valid reservation map pointer where
         * it is being copied for another still existing VMA.  As that VMA
         * has a reference to the reservation map it cannot disappear until
         * after this open call completes.  It is therefore safe to take a
         * new reference here without additional locking.
         */
        if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
                resv_map_dup_hugetlb_cgroup_uncharge_info(resv);
                kref_get(&resv->refs);
        }

        /*
         * vma_lock structure for sharable mappings is vma specific.
         * Clear old pointer (if copied via vm_area_dup) and allocate
         * new structure.  Before clearing, make sure vma_lock is not
         * for this vma.
         */
        if (vma->vm_flags & VM_MAYSHARE) {
                struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

                if (vma_lock) {
                        if (vma_lock->vma != vma) {
                                vma->vm_private_data = NULL;
                                hugetlb_vma_lock_alloc(vma);
                        } else {
                                pr_warn("HugeTLB: vma_lock already exists in %s.\n", __func__);
                        }
                } else {
                        hugetlb_vma_lock_alloc(vma);
                }
        }
}

static void hugetlb_vm_op_close(struct vm_area_struct *vma)
{
        struct hstate *h = hstate_vma(vma);
        struct resv_map *resv;
        struct hugepage_subpool *spool = subpool_vma(vma);
        unsigned long reserve, start, end;
        long gbl_reserve;

        hugetlb_vma_lock_free(vma);

        resv = vma_resv_map(vma);
        if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
                return;

        start = vma_hugecache_offset(h, vma, vma->vm_start);
        end = vma_hugecache_offset(h, vma, vma->vm_end);

        reserve = (end - start) - region_count(resv, start, end);
        hugetlb_cgroup_uncharge_counter(resv, start, end);
        if (reserve) {
                /*
                 * Decrement reserve counts.  The global reserve count may be
                 * adjusted if the subpool has a minimum size.
                 */
                gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
                hugetlb_acct_memory(h, -gbl_reserve);
        }

        kref_put(&resv->refs, resv_map_release);
}

static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
{
        if (addr & ~(huge_page_mask(hstate_vma(vma))))
                return -EINVAL;
        return 0;
}

void hugetlb_split(struct vm_area_struct *vma, unsigned long addr)
{
        /*
         * PMD sharing is only possible for PUD_SIZE-aligned address ranges
         * in HugeTLB VMAs. If we will lose PUD_SIZE alignment due to this
         * split, unshare PMDs in the PUD_SIZE interval surrounding addr now.
         * This function is called in the middle of a VMA split operation, with
         * MM, VMA and rmap all write-locked to prevent concurrent page table
         * walks (except hardware and gup_fast()).
         */
        vma_assert_write_locked(vma);
        i_mmap_assert_write_locked(vma->vm_file->f_mapping);

        if (addr & ~PUD_MASK) {
                unsigned long floor = addr & PUD_MASK;
                unsigned long ceil = floor + PUD_SIZE;

                if (floor >= vma->vm_start && ceil <= vma->vm_end) {
                        /*
                         * Locking:
                         * Use take_locks=false here.
                         * The file rmap lock is already held.
                         * The hugetlb VMA lock can't be taken when we already
                         * hold the file rmap lock, and we don't need it because
                         * its purpose is to synchronize against concurrent page
                         * table walks, which are not possible thanks to the
                         * locks held by our caller.
                         */
                        hugetlb_unshare_pmds(vma, floor, ceil, /* take_locks = */ false);
                }
        }
}

static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
{
        return huge_page_size(hstate_vma(vma));
}

/*
 * We cannot handle pagefaults against hugetlb pages at all.  They cause
 * handle_mm_fault() to try to instantiate regular-sized pages in the
 * hugepage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
 * this far.
 */
static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
{
        BUG();
        return 0;
}

#ifdef CONFIG_USERFAULTFD
static bool hugetlb_can_userfault(struct vm_area_struct *vma,
                                  vm_flags_t vm_flags)
{
        return true;
}

static const struct vm_uffd_ops hugetlb_uffd_ops = {
        .can_userfault = hugetlb_can_userfault,
};
#endif

/*
 * When a new function is introduced to vm_operations_struct and added
 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
 * This is because under System V memory model, mappings created via
 * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
 * their original vm_ops are overwritten with shm_vm_ops.
 */
const struct vm_operations_struct hugetlb_vm_ops = {
        .fault = hugetlb_vm_op_fault,
        .open = hugetlb_vm_op_open,
        .close = hugetlb_vm_op_close,
        .may_split = hugetlb_vm_op_split,
        .pagesize = hugetlb_vm_op_pagesize,
#ifdef CONFIG_USERFAULTFD
        .uffd_ops = &hugetlb_uffd_ops,
#endif
};

static pte_t make_huge_pte(struct vm_area_struct *vma, struct folio *folio,
                bool try_mkwrite)
{
        pte_t entry = folio_mk_pte(folio, vma->vm_page_prot);
        unsigned int shift = huge_page_shift(hstate_vma(vma));

        if (try_mkwrite && (vma->vm_flags & VM_WRITE)) {
                entry = pte_mkwrite_novma(pte_mkdirty(entry));
        } else {
                entry = pte_wrprotect(entry);
        }
        entry = pte_mkyoung(entry);
        entry = arch_make_huge_pte(entry, shift, vma->vm_flags);

        return entry;
}

static void set_huge_ptep_writable(struct vm_area_struct *vma,
                                   unsigned long address, pte_t *ptep)
{
        pte_t entry;

        entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(vma->vm_mm, address, ptep)));
        if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
                update_mmu_cache(vma, address, ptep);
}

static void set_huge_ptep_maybe_writable(struct vm_area_struct *vma,
                                         unsigned long address, pte_t *ptep)
{
        if (vma->vm_flags & VM_WRITE)
                set_huge_ptep_writable(vma, address, ptep);
}

static void
hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
                      struct folio *new_folio, pte_t old, unsigned long sz)
{
        pte_t newpte = make_huge_pte(vma, new_folio, true);

        __folio_mark_uptodate(new_folio);
        hugetlb_add_new_anon_rmap(new_folio, vma, addr);
        if (userfaultfd_protected(vma) && huge_pte_uffd(old)) {
                newpte = huge_pte_mkuffd(newpte);
                /* Restore PAGE_NONE so the RWP marker keeps trapping. */
                if (userfaultfd_rwp(vma)) {
                        unsigned int shift = huge_page_shift(hstate_vma(vma));

                        newpte = huge_pte_modify(newpte, PAGE_NONE);
                        newpte = arch_make_huge_pte(newpte, shift, vma->vm_flags);
                }
        }
        set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz);
        hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
        folio_set_hugetlb_migratable(new_folio);
}

int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
                            struct vm_area_struct *dst_vma,
                            struct vm_area_struct *src_vma)
{
        pte_t *src_pte, *dst_pte, entry;
        struct folio *pte_folio;
        unsigned long addr;
        bool cow = vma_is_cow_mapping(src_vma);
        struct hstate *h = hstate_vma(src_vma);
        unsigned long sz = huge_page_size(h);
        unsigned long npages = pages_per_huge_page(h);
        struct mmu_notifier_range range;
        unsigned long last_addr_mask;
        softleaf_t softleaf;
        int ret = 0;

        if (cow) {
                mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src,
                                        src_vma->vm_start,
                                        src_vma->vm_end);
                mmu_notifier_invalidate_range_start(&range);
                vma_assert_write_locked(src_vma);
                raw_write_seqcount_begin(&src->write_protect_seq);
        } else {
                /*
                 * For shared mappings the vma lock must be held before
                 * calling hugetlb_walk() in the src vma. Otherwise, the
                 * returned ptep could go away if part of a shared pmd and
                 * another thread calls huge_pmd_unshare.
                 */
                hugetlb_vma_lock_read(src_vma);
        }

        last_addr_mask = hugetlb_mask_last_page(h);
        for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) {
                spinlock_t *src_ptl, *dst_ptl;
                src_pte = hugetlb_walk(src_vma, addr, sz);
                if (!src_pte) {
                        addr |= last_addr_mask;
                        continue;
                }
                dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz);
                if (!dst_pte) {
                        ret = -ENOMEM;
                        break;
                }

#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
                /* If the pagetables are shared, there is nothing to do */
                if (ptdesc_pmd_is_shared(virt_to_ptdesc(dst_pte))) {
                        addr |= last_addr_mask;
                        continue;
                }
#endif

                dst_ptl = huge_pte_lock(h, dst, dst_pte);
                src_ptl = huge_pte_lockptr(h, src, src_pte);
                spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
                entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte);
again:
                if (huge_pte_none(entry)) {
                        /* Skip if src entry none. */
                        goto next;
                }

                softleaf = softleaf_from_pte(entry);
                if (unlikely(softleaf_is_hwpoison(softleaf))) {
                        /*
                         * A hwpoison entry never carries the uffd-wp bit: it is
                         * installed fresh by make_hwpoison_entry() and
                         * hugetlb_change_protection() leaves it untouched, so
                         * there is nothing to clear for the child.
                         */
                        set_huge_pte_at(dst, addr, dst_pte, entry, sz);
                } else if (unlikely(softleaf_is_migration(softleaf))) {
                        bool uffd = pte_swp_uffd(entry);

                        if (!softleaf_is_migration_read(softleaf) && cow) {
                                /*
                                 * COW mappings require pages in both
                                 * parent and child to be set to read.
                                 */
                                softleaf = make_readable_migration_entry(
                                                        swp_offset(softleaf));
                                entry = swp_entry_to_pte(softleaf);
                                if (userfaultfd_protected(src_vma) && uffd)
                                        entry = pte_swp_mkuffd(entry);
                                set_huge_pte_at(src, addr, src_pte, entry, sz);
                        }
                        if (!userfaultfd_protected(dst_vma))
                                entry = pte_swp_clear_uffd(entry);
                        set_huge_pte_at(dst, addr, dst_pte, entry, sz);
                } else if (unlikely(pte_is_marker(entry))) {
                        const pte_marker marker = copy_pte_marker(softleaf, dst_vma);

                        if (marker)
                                set_huge_pte_at(dst, addr, dst_pte,
                                                make_pte_marker(marker), sz);
                } else {
                        entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte);
                        pte_folio = page_folio(pte_page(entry));
                        folio_get(pte_folio);

                        /*
                         * Failing to duplicate the anon rmap is a rare case
                         * where we see pinned hugetlb pages while they're
                         * prone to COW. We need to do the COW earlier during
                         * fork.
                         *
                         * When pre-allocating the page or copying data, we
                         * need to be without the pgtable locks since we could
                         * sleep during the process.
                         */
                        if (!folio_test_anon(pte_folio)) {
                                hugetlb_add_file_rmap(pte_folio);
                        } else if (hugetlb_try_dup_anon_rmap(pte_folio, src_vma)) {
                                pte_t src_pte_old = entry;
                                struct folio *new_folio;

                                spin_unlock(src_ptl);
                                spin_unlock(dst_ptl);
                                /* Do not use reserve as it's private owned */
                                new_folio = alloc_hugetlb_folio(dst_vma, addr, false);
                                if (IS_ERR(new_folio)) {
                                        folio_put(pte_folio);
                                        ret = PTR_ERR(new_folio);
                                        break;
                                }
                                ret = copy_user_large_folio(new_folio, pte_folio,
                                                            addr, dst_vma);
                                folio_put(pte_folio);
                                if (ret) {
                                        restore_reserve_on_error(h, dst_vma, addr, new_folio);
                                        folio_put(new_folio);
                                        break;
                                }

                                /* Install the new hugetlb folio if src pte stable */
                                dst_ptl = huge_pte_lock(h, dst, dst_pte);
                                src_ptl = huge_pte_lockptr(h, src, src_pte);
                                spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
                                entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte);
                                if (!pte_same(src_pte_old, entry)) {
                                        restore_reserve_on_error(h, dst_vma, addr,
                                                                new_folio);
                                        folio_put(new_folio);
                                        /* huge_ptep of dst_pte won't change as in child */
                                        goto again;
                                }
                                hugetlb_install_folio(dst_vma, dst_pte, addr,
                                                      new_folio, src_pte_old, sz);
                                goto next;
                        }

                        /* See __copy_present_ptes(): restore accessible protection. */
                        if (!userfaultfd_protected(dst_vma)) {
                                if (userfaultfd_rwp(src_vma) && huge_pte_uffd(entry)) {
                                        entry = huge_pte_modify(entry, dst_vma->vm_page_prot);
                                        entry = arch_make_huge_pte(entry, huge_page_shift(h),
                                                                   dst_vma->vm_flags);
                                }
                                entry = huge_pte_clear_uffd(entry);
                        }

                        if (cow) {
                                /*
                                 * No need to notify as we are downgrading page
                                 * table protection not changing it to point
                                 * to a new page.
                                 *
                                 * See Documentation/mm/mmu_notifier.rst
                                 */
                                huge_ptep_set_wrprotect(src, addr, src_pte);
                                entry = huge_pte_wrprotect(entry);
                        }

                        set_huge_pte_at(dst, addr, dst_pte, entry, sz);
                        hugetlb_count_add(npages, dst);
                }

next:
                spin_unlock(src_ptl);
                spin_unlock(dst_ptl);
        }

        if (cow) {
                raw_write_seqcount_end(&src->write_protect_seq);
                mmu_notifier_invalidate_range_end(&range);
        } else {
                hugetlb_vma_unlock_read(src_vma);
        }

        return ret;
}

static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
                          unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte,
                          unsigned long sz)
{
        bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
        struct hstate *h = hstate_vma(vma);
        struct mm_struct *mm = vma->vm_mm;
        spinlock_t *src_ptl, *dst_ptl;
        pte_t pte;

        dst_ptl = huge_pte_lock(h, mm, dst_pte);
        src_ptl = huge_pte_lockptr(h, mm, src_pte);

        /*
         * We don't have to worry about the ordering of src and dst ptlocks
         * because exclusive mmap_lock (or the i_mmap_lock) prevents deadlock.
         */
        if (src_ptl != dst_ptl)
                spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);

        pte = huge_ptep_get_and_clear(mm, old_addr, src_pte, sz);

        if (need_clear_uffd_wp && pte_is_uffd_wp_marker(pte)) {
                huge_pte_clear(mm, new_addr, dst_pte, sz);
        } else {
                if (need_clear_uffd_wp) {
                        if (pte_present(pte)) {
                                /*
                                 * See __copy_present_ptes(): normalise the RWP
                                 * marker so the destination starts accessible
                                 * instead of taking a numa-hinting fault on
                                 * first access. Only the marker (protnone + uffd)
                                 * needs it; leave other present PTEs untouched.
                                 */
                                if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) {
                                        pte = huge_pte_modify(pte, vma->vm_page_prot);
                                        pte = arch_make_huge_pte(pte, huge_page_shift(h),
                                                                 vma->vm_flags);
                                }
                                pte = huge_pte_clear_uffd(pte);
                        } else {
                                pte = pte_swp_clear_uffd(pte);
                        }
                }
                set_huge_pte_at(mm, new_addr, dst_pte, pte, sz);
        }

        if (src_ptl != dst_ptl)
                spin_unlock(src_ptl);
        spin_unlock(dst_ptl);
}

int move_hugetlb_page_tables(struct vm_area_struct *vma,
                             struct vm_area_struct *new_vma,
                             unsigned long old_addr, unsigned long new_addr,
                             unsigned long len)
{
        struct hstate *h = hstate_vma(vma);
        struct address_space *mapping = vma->vm_file->f_mapping;
        unsigned long sz = huge_page_size(h);
        struct mm_struct *mm = vma->vm_mm;
        unsigned long old_end = old_addr + len;
        unsigned long last_addr_mask;
        pte_t *src_pte, *dst_pte;
        struct mmu_notifier_range range;
        struct mmu_gather tlb;

        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, old_addr,
                                old_end);
        adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
        /*
         * In case of shared PMDs, we should cover the maximum possible
         * range.
         */
        flush_cache_range(vma, range.start, range.end);
        tlb_gather_mmu_vma(&tlb, vma);

        mmu_notifier_invalidate_range_start(&range);
        last_addr_mask = hugetlb_mask_last_page(h);
        /* Prevent race with file truncation */
        hugetlb_vma_lock_write(vma);
        i_mmap_lock_write(mapping);
        for (; old_addr < old_end; old_addr += sz, new_addr += sz) {
                src_pte = hugetlb_walk(vma, old_addr, sz);
                if (!src_pte) {
                        old_addr |= last_addr_mask;
                        new_addr |= last_addr_mask;
                        continue;
                }
                if (huge_pte_none(huge_ptep_get(mm, old_addr, src_pte)))
                        continue;

                if (huge_pmd_unshare(&tlb, vma, old_addr, src_pte)) {
                        old_addr |= last_addr_mask;
                        new_addr |= last_addr_mask;
                        continue;
                }

                dst_pte = huge_pte_alloc(mm, new_vma, new_addr, sz);
                if (!dst_pte)
                        break;

                move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte, sz);
                tlb_remove_huge_tlb_entry(h, &tlb, src_pte, old_addr);
        }

        tlb_flush_mmu_tlbonly(&tlb);
        huge_pmd_unshare_flush(&tlb, vma);

        mmu_notifier_invalidate_range_end(&range);
        i_mmap_unlock_write(mapping);
        hugetlb_vma_unlock_write(vma);
        tlb_finish_mmu(&tlb);

        return len + old_addr - old_end;
}

void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
                            unsigned long start, unsigned long end,
                            struct folio *folio, zap_flags_t zap_flags)
{
        struct mm_struct *mm = vma->vm_mm;
        const bool folio_provided = !!folio;
        unsigned long address;
        pte_t *ptep;
        pte_t pte;
        spinlock_t *ptl;
        struct hstate *h = hstate_vma(vma);
        unsigned long sz = huge_page_size(h);
        bool adjust_reservation;
        unsigned long last_addr_mask;

        i_mmap_assert_write_locked(vma->vm_file->f_mapping);
        WARN_ON(!is_vm_hugetlb_page(vma));
        BUG_ON(start & ~huge_page_mask(h));
        BUG_ON(end & ~huge_page_mask(h));

        /*
         * This is a hugetlb vma, all the pte entries should point
         * to huge page.
         */
        tlb_change_page_size(tlb, sz);
        tlb_start_vma(tlb, vma);

        last_addr_mask = hugetlb_mask_last_page(h);
        address = start;
        for (; address < end; address += sz) {
                ptep = hugetlb_walk(vma, address, sz);
                if (!ptep) {
                        address |= last_addr_mask;
                        continue;
                }

                ptl = huge_pte_lock(h, mm, ptep);
                if (huge_pmd_unshare(tlb, vma, address, ptep)) {
                        spin_unlock(ptl);
                        address |= last_addr_mask;
                        continue;
                }

                pte = huge_ptep_get(mm, address, ptep);
                if (huge_pte_none(pte)) {
                        spin_unlock(ptl);
                        continue;
                }

                /*
                 * Migrating hugepage or HWPoisoned hugepage is already
                 * unmapped and its refcount is dropped, so just clear pte here.
                 */
                if (unlikely(!pte_present(pte))) {
                        /*
                         * If the pte was wr-protected by uffd-wp in any of the
                         * swap forms, meanwhile the caller does not want to
                         * drop the uffd-wp bit in this zap, then replace the
                         * pte with a marker.
                         */
                        if (pte_swp_uffd_any(pte) &&
                            !(zap_flags & ZAP_FLAG_DROP_MARKER))
                                set_huge_pte_at(mm, address, ptep,
                                                make_pte_marker(PTE_MARKER_UFFD_WP),
                                                sz);
                        else
                                huge_pte_clear(mm, address, ptep, sz);
                        spin_unlock(ptl);
                        continue;
                }

                /*
                 * If a folio is supplied, it is because a specific
                 * folio is being unmapped, not a range. Ensure the folio we
                 * are about to unmap is the actual folio of interest.
                 */
                if (folio_provided) {
                        if (folio != page_folio(pte_page(pte))) {
                                spin_unlock(ptl);
                                continue;
                        }
                        /*
                         * Mark the VMA as having unmapped its page so that
                         * future faults in this VMA will fail rather than
                         * looking like data was lost
                         */
                        set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
                } else {
                        folio = page_folio(pte_page(pte));
                }

                pte = huge_ptep_get_and_clear(mm, address, ptep, sz);
                tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
                if (huge_pte_dirty(pte))
                        folio_mark_dirty(folio);
                /* Leave a uffd-wp pte marker if needed */
                if (huge_pte_uffd(pte) &&
                    !(zap_flags & ZAP_FLAG_DROP_MARKER))
                        set_huge_pte_at(mm, address, ptep,
                                        make_pte_marker(PTE_MARKER_UFFD_WP),
                                        sz);
                hugetlb_count_sub(pages_per_huge_page(h), mm);
                hugetlb_remove_rmap(folio);
                spin_unlock(ptl);

                /*
                 * Restore the reservation for anonymous page, otherwise the
                 * backing page could be stolen by someone. Restore only on the
                 * last unmap, otherwise the owner could empty its resv map
                 * while the folio is still mapped by a child. Note that holding
                 * i_mmap_lock_write is needed to check the number of mappings.
                 * If there we are freeing a surplus, do not set the restore
                 * reservation bit.
                 */
                adjust_reservation = false;

                spin_lock_irq(&hugetlb_lock);
                if (!h->surplus_huge_pages && __vma_private_lock(vma) &&
                    !folio_mapped(folio) && folio_test_anon(folio)) {
                        folio_set_hugetlb_restore_reserve(folio);
                        /* Reservation to be adjusted after the spin lock */
                        adjust_reservation = true;
                }
                spin_unlock_irq(&hugetlb_lock);

                /*
                 * Adjust the reservation for the region that will have the
                 * reserve restored. Keep in mind that vma_needs_reservation() changes
                 * resv->adds_in_progress if it succeeds. If this is not done,
                 * do_exit() will not see it, and will keep the reservation
                 * forever.
                 */
                if (adjust_reservation) {
                        int rc = vma_needs_reservation(h, vma, address);

                        if (rc < 0)
                                /* Pressumably allocate_file_region_entries failed
                                 * to allocate a file_region struct. Clear
                                 * hugetlb_restore_reserve so that global reserve
                                 * count will not be incremented by free_huge_folio.
                                 * Act as if we consumed the reservation.
                                 */
                                folio_clear_hugetlb_restore_reserve(folio);
                        else if (rc)
                                vma_add_reservation(h, vma, address);
                }

                tlb_remove_page_size(tlb, folio_page(folio, 0),
                                     folio_size(folio));
                /*
                 * If we were instructed to unmap a specific folio, we're done.
                 */
                if (folio_provided)
                        break;
        }
        tlb_end_vma(tlb, vma);

        huge_pmd_unshare_flush(tlb, vma);
}

void __hugetlb_zap_begin(struct vm_area_struct *vma,
                         unsigned long *start, unsigned long *end)
{
        if (!vma->vm_file)        /* hugetlbfs_file_mmap error */
                return;

        adjust_range_if_pmd_sharing_possible(vma, start, end);
        hugetlb_vma_lock_write(vma);
        if (vma->vm_file)
                i_mmap_lock_write(vma->vm_file->f_mapping);
}

void __hugetlb_zap_end(struct vm_area_struct *vma,
                       struct zap_details *details)
{
        zap_flags_t zap_flags = details ? details->zap_flags : 0;

        if (!vma->vm_file)        /* hugetlbfs_file_mmap error */
                return;

        if (zap_flags & ZAP_FLAG_UNMAP) {        /* final unmap */
                /*
                 * Unlock and free the vma lock before releasing i_mmap_rwsem.
                 * When the vma_lock is freed, this makes the vma ineligible
                 * for pmd sharing.  And, i_mmap_rwsem is required to set up
                 * pmd sharing.  This is important as page tables for this
                 * unmapped range will be asynchrously deleted.  If the page
                 * tables are shared, there will be issues when accessed by
                 * someone else.
                 */
                __hugetlb_vma_unlock_write_free(vma);
        } else {
                hugetlb_vma_unlock_write(vma);
        }

        if (vma->vm_file)
                i_mmap_unlock_write(vma->vm_file->f_mapping);
}

void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
                          unsigned long end, struct folio *folio,
                          zap_flags_t zap_flags)
{
        struct mmu_notifier_range range;
        struct mmu_gather tlb;

        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
                                start, end);
        adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
        mmu_notifier_invalidate_range_start(&range);
        tlb_gather_mmu(&tlb, vma->vm_mm);

        __unmap_hugepage_range(&tlb, vma, start, end,
                               folio, zap_flags);

        mmu_notifier_invalidate_range_end(&range);
        tlb_finish_mmu(&tlb);
}

/*
 * This is called when the original mapper is failing to COW a MAP_PRIVATE
 * mapping it owns the reserve page for. The intention is to unmap the page
 * from other VMAs and let the children be SIGKILLed if they are faulting the
 * same region.
 */
static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
                              struct folio *folio, unsigned long address)
{
        struct hstate *h = hstate_vma(vma);
        struct vm_area_struct *iter_vma;
        struct address_space *mapping;
        pgoff_t pgoff;

        /*
         * vm_pgoff is in PAGE_SIZE units, hence the different calculation
         * from page cache lookup which is in HPAGE_SIZE units.
         */
        address = address & huge_page_mask(h);
        pgoff = linear_page_index(vma, address);
        mapping = vma->vm_file->f_mapping;

        /*
         * Take the mapping lock for the duration of the table walk. As
         * this mapping should be shared between all the VMAs,
         * __unmap_hugepage_range() is called as the lock is already held
         */
        i_mmap_lock_write(mapping);
        mapping_rmap_tree_foreach(iter_vma, mapping, pgoff, pgoff) {
                /* Do not unmap the current VMA */
                if (iter_vma == vma)
                        continue;

                /*
                 * Shared VMAs have their own reserves and do not affect
                 * MAP_PRIVATE accounting but it is possible that a shared
                 * VMA is using the same page so check and skip such VMAs.
                 */
                if (iter_vma->vm_flags & VM_MAYSHARE)
                        continue;

                /*
                 * Unmap the page from other VMAs without their own reserves.
                 * They get marked to be SIGKILLed if they fault in these
                 * areas. This is because a future no-page fault on this VMA
                 * could insert a zeroed page instead of the data existing
                 * from the time of fork. This would look like data corruption
                 */
                if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
                        unmap_hugepage_range(iter_vma, address,
                                             address + huge_page_size(h),
                                             folio, 0);
        }
        i_mmap_unlock_write(mapping);
}

/*
 * hugetlb_wp() should be called with page lock of the original hugepage held.
 * Called with hugetlb_fault_mutex_table held and pte_page locked so we
 * cannot race with other handlers or page migration.
 * Keep the pte_same checks anyway to make transition from the mutex easier.
 */
static vm_fault_t hugetlb_wp(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct mm_struct *mm = vma->vm_mm;
        const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
        pte_t pte = huge_ptep_get(mm, vmf->address, vmf->pte);
        struct hstate *h = hstate_vma(vma);
        struct folio *old_folio;
        struct folio *new_folio;
        bool cow_from_owner = 0;
        vm_fault_t ret = 0;
        struct mmu_notifier_range range;

        /*
         * Never handle CoW for uffd-wp protected pages.  It should be only
         * handled when the uffd-wp protection is removed.
         *
         * Note that only the CoW optimization path (in hugetlb_no_page())
         * can trigger this, because hugetlb_fault() will always resolve
         * uffd-wp bit first.
         */
        if (!unshare && huge_pte_uffd(pte))
                return 0;

        /* Let's take out MAP_SHARED mappings first. */
        if (vma->vm_flags & VM_MAYSHARE) {
                set_huge_ptep_writable(vma, vmf->address, vmf->pte);
                return 0;
        }

        old_folio = page_folio(pte_page(pte));

        delayacct_wpcopy_start();

retry_avoidcopy:
        /*
         * If no-one else is actually using this page, we're the exclusive
         * owner and can reuse this page.
         *
         * Note that we don't rely on the (safer) folio refcount here, because
         * copying the hugetlb folio when there are unexpected (temporary)
         * folio references could harm simple fork()+exit() users when
         * we run out of free hugetlb folios: we would have to kill processes
         * in scenarios that used to work. As a side effect, there can still
         * be leaks between processes, for example, with FOLL_GET users.
         */
        if (folio_mapcount(old_folio) == 1 && folio_test_anon(old_folio)) {
                if (!PageAnonExclusive(&old_folio->page)) {
                        folio_move_anon_rmap(old_folio, vma);
                        SetPageAnonExclusive(&old_folio->page);
                }
                if (likely(!unshare))
                        set_huge_ptep_maybe_writable(vma, vmf->address,
                                                     vmf->pte);

                delayacct_wpcopy_end();
                return 0;
        }
        VM_BUG_ON_PAGE(folio_test_anon(old_folio) &&
                       PageAnonExclusive(&old_folio->page), &old_folio->page);

        /*
         * If the process that created a MAP_PRIVATE mapping is about to perform
         * a COW due to a shared page count, attempt to satisfy the allocation
         * without using the existing reserves.
         * In order to determine where this is a COW on a MAP_PRIVATE mapping it
         * is enough to check whether the old_folio is anonymous. This means that
         * the reserve for this address was consumed. If reserves were used, a
         * partial faulted mapping at the fime of fork() could consume its reserves
         * on COW instead of the full address range.
         */
        if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
            folio_test_anon(old_folio))
                cow_from_owner = true;

        folio_get(old_folio);

        /*
         * Drop page table lock as buddy allocator may be called. It will
         * be acquired again before returning to the caller, as expected.
         */
        spin_unlock(vmf->ptl);
        new_folio = alloc_hugetlb_folio(vma, vmf->address, cow_from_owner);

        if (IS_ERR(new_folio)) {
                /*
                 * If a process owning a MAP_PRIVATE mapping fails to COW,
                 * it is due to references held by a child and an insufficient
                 * huge page pool. To guarantee the original mappers
                 * reliability, unmap the page from child processes. The child
                 * may get SIGKILLed if it later faults.
                 */
                if (cow_from_owner) {
                        struct address_space *mapping = vma->vm_file->f_mapping;
                        pgoff_t idx;
                        u32 hash;

                        folio_put(old_folio);
                        /*
                         * Drop hugetlb_fault_mutex and vma_lock before
                         * unmapping.  unmapping needs to hold vma_lock
                         * in write mode.  Dropping vma_lock in read mode
                         * here is OK as COW mappings do not interact with
                         * PMD sharing.
                         *
                         * Reacquire both after unmap operation.
                         */
                        idx = vma_hugecache_offset(h, vma, vmf->address);
                        hash = hugetlb_fault_mutex_hash(mapping, idx);
                        hugetlb_vma_unlock_read(vma);
                        mutex_unlock(&hugetlb_fault_mutex_table[hash]);

                        unmap_ref_private(mm, vma, old_folio, vmf->address);

                        mutex_lock(&hugetlb_fault_mutex_table[hash]);
                        hugetlb_vma_lock_read(vma);
                        spin_lock(vmf->ptl);
                        vmf->pte = hugetlb_walk(vma, vmf->address,
                                        huge_page_size(h));
                        if (likely(vmf->pte &&
                                   pte_same(huge_ptep_get(mm, vmf->address, vmf->pte), pte)))
                                goto retry_avoidcopy;
                        /*
                         * race occurs while re-acquiring page table
                         * lock, and our job is done.
                         */
                        delayacct_wpcopy_end();
                        return 0;
                }

                ret = vmf_error(PTR_ERR(new_folio));
                goto out_release_old;
        }

        /*
         * When the original hugepage is shared one, it does not have
         * anon_vma prepared.
         */
        ret = __vmf_anon_prepare(vmf);
        if (unlikely(ret))
                goto out_release_all;

        if (copy_user_large_folio(new_folio, old_folio, vmf->real_address, vma)) {
                ret = VM_FAULT_HWPOISON_LARGE | VM_FAULT_SET_HINDEX(hstate_index(h));
                goto out_release_all;
        }
        __folio_mark_uptodate(new_folio);

        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, vmf->address,
                                vmf->address + huge_page_size(h));
        mmu_notifier_invalidate_range_start(&range);

        /*
         * Retake the page table lock to check for racing updates
         * before the page tables are altered
         */
        spin_lock(vmf->ptl);
        vmf->pte = hugetlb_walk(vma, vmf->address, huge_page_size(h));
        if (likely(vmf->pte && pte_same(huge_ptep_get(mm, vmf->address, vmf->pte), pte))) {
                pte_t newpte = make_huge_pte(vma, new_folio, !unshare);

                /* Break COW or unshare */
                huge_ptep_clear_flush(vma, vmf->address, vmf->pte);
                hugetlb_remove_rmap(old_folio);
                hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address);
                if (huge_pte_uffd(pte))
                        newpte = huge_pte_mkuffd(newpte);
                set_huge_pte_at(mm, vmf->address, vmf->pte, newpte,
                                huge_page_size(h));
                folio_set_hugetlb_migratable(new_folio);
                /* Make the old page be freed below */
                new_folio = old_folio;
        }
        spin_unlock(vmf->ptl);
        mmu_notifier_invalidate_range_end(&range);
out_release_all:
        /*
         * No restore in case of successful pagetable update (Break COW or
         * unshare)
         */
        if (new_folio != old_folio)
                restore_reserve_on_error(h, vma, vmf->address, new_folio);
        folio_put(new_folio);
out_release_old:
        folio_put(old_folio);

        spin_lock(vmf->ptl); /* Caller expects lock to be held */

        delayacct_wpcopy_end();
        return ret;
}

/*
 * Return whether there is a pagecache page to back given address within VMA.
 */
bool hugetlbfs_pagecache_present(struct hstate *h,
                                 struct vm_area_struct *vma, unsigned long address)
{
        struct address_space *mapping = vma->vm_file->f_mapping;
        pgoff_t idx = linear_page_index(vma, address);
        struct folio *folio;

        folio = filemap_get_folio(mapping, idx);
        if (IS_ERR(folio))
                return false;
        folio_put(folio);
        return true;
}

int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping,
                           pgoff_t idx)
{
        struct inode *inode = mapping->host;
        struct hstate *h = hstate_inode(inode);
        int err;

        idx <<= huge_page_order(h);
        __folio_set_locked(folio);
        err = __filemap_add_folio(mapping, folio, idx, GFP_KERNEL, NULL);

        if (unlikely(err)) {
                __folio_clear_locked(folio);
                return err;
        }
        folio_clear_hugetlb_restore_reserve(folio);

        /*
         * mark folio dirty so that it will not be removed from cache/file
         * by non-hugetlbfs specific code paths.
         */
        folio_mark_dirty(folio);

        spin_lock(&inode->i_lock);
        inode->i_blocks += blocks_per_huge_page(h);
        spin_unlock(&inode->i_lock);
        return 0;
}

static inline vm_fault_t hugetlb_handle_userfault(struct vm_fault *vmf,
                                                  struct address_space *mapping,
                                                  unsigned long reason)
{
        u32 hash;

        /*
         * vma_lock and hugetlb_fault_mutex must be dropped before handling
         * userfault. Also mmap_lock could be dropped due to handling
         * userfault, any vma operation should be careful from here.
         */
        hugetlb_vma_unlock_read(vmf->vma);
        hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff);
        mutex_unlock(&hugetlb_fault_mutex_table[hash]);
        return handle_userfault(vmf, reason);
}

/*
 * Recheck pte with pgtable lock.  Returns true if pte didn't change, or
 * false if pte changed or is changing.
 */
static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm, unsigned long addr,
                               pte_t *ptep, pte_t old_pte)
{
        spinlock_t *ptl;
        bool same;

        ptl = huge_pte_lock(h, mm, ptep);
        same = pte_same(huge_ptep_get(mm, addr, ptep), old_pte);
        spin_unlock(ptl);

        return same;
}

static vm_fault_t hugetlb_no_page(struct address_space *mapping,
                        struct vm_fault *vmf)
{
        u32 hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff);
        bool new_folio, new_anon_folio = false;
        struct vm_area_struct *vma = vmf->vma;
        struct mm_struct *mm = vma->vm_mm;
        struct hstate *h = hstate_vma(vma);
        vm_fault_t ret = VM_FAULT_SIGBUS;
        bool folio_locked = true;
        struct folio *folio;
        unsigned long size;
        pte_t new_pte;

        /*
         * Currently, we are forced to kill the process in the event the
         * original mapper has unmapped pages from the child due to a failed
         * COW/unsharing. Warn that such a situation has occurred as it may not
         * be obvious.
         */
        if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
                pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
                           current->pid);
                goto out;
        }

        /*
         * Use page lock to guard against racing truncation
         * before we get page_table_lock.
         */
        new_folio = false;
        folio = filemap_lock_hugetlb_folio(h, mapping, vmf->pgoff);
        if (IS_ERR(folio)) {
                size = i_size_read(mapping->host) >> huge_page_shift(h);
                if (vmf->pgoff >= size)
                        goto out;
                /* Check for page in userfault range */
                if (userfaultfd_missing(vma)) {
                        /*
                         * Since hugetlb_no_page() was examining pte
                         * without pgtable lock, we need to re-test under
                         * lock because the pte may not be stable and could
                         * have changed from under us.  Try to detect
                         * either changed or during-changing ptes and retry
                         * properly when needed.
                         *
                         * Note that userfaultfd is actually fine with
                         * false positives (e.g. caused by pte changed),
                         * but not wrong logical events (e.g. caused by
                         * reading a pte during changing).  The latter can
                         * confuse the userspace, so the strictness is very
                         * much preferred.  E.g., MISSING event should
                         * never happen on the page after UFFDIO_COPY has
                         * correctly installed the page and returned.
                         */
                        if (!hugetlb_pte_stable(h, mm, vmf->address, vmf->pte, vmf->orig_pte)) {
                                ret = 0;
                                goto out;
                        }

                        return hugetlb_handle_userfault(vmf, mapping,
                                                        VM_UFFD_MISSING);
                }

                if (!(vma->vm_flags & VM_MAYSHARE)) {
                        ret = __vmf_anon_prepare(vmf);
                        if (unlikely(ret))
                                goto out;
                }

                folio = alloc_hugetlb_folio(vma, vmf->address, false);
                if (IS_ERR(folio)) {
                        /*
                         * Returning error will result in faulting task being
                         * sent SIGBUS.  The hugetlb fault mutex prevents two
                         * tasks from racing to fault in the same page which
                         * could result in false unable to allocate errors.
                         * Page migration does not take the fault mutex, but
                         * does a clear then write of pte's under page table
                         * lock.  Page fault code could race with migration,
                         * notice the clear pte and try to allocate a page
                         * here.  Before returning error, get ptl and make
                         * sure there really is no pte entry.
                         */
                        if (hugetlb_pte_stable(h, mm, vmf->address, vmf->pte, vmf->orig_pte))
                                ret = vmf_error(PTR_ERR(folio));
                        else
                                ret = 0;
                        goto out;
                }
                folio_zero_user(folio, vmf->real_address);
                __folio_mark_uptodate(folio);
                new_folio = true;

                if (vma->vm_flags & VM_MAYSHARE) {
                        int err = hugetlb_add_to_page_cache(folio, mapping,
                                                        vmf->pgoff);
                        if (err) {
                                /*
                                 * err can't be -EEXIST which implies someone
                                 * else consumed the reservation since hugetlb
                                 * fault mutex is held when add a hugetlb page
                                 * to the page cache. So it's safe to call
                                 * restore_reserve_on_error() here.
                                 */
                                restore_reserve_on_error(h, vma, vmf->address,
                                                        folio);
                                folio_put(folio);
                                ret = VM_FAULT_SIGBUS;
                                goto out;
                        }
                } else {
                        new_anon_folio = true;
                        folio_lock(folio);
                }
        } else {
                /*
                 * If memory error occurs between mmap() and fault, some process
                 * don't have hwpoisoned swap entry for errored virtual address.
                 * So we need to block hugepage fault by PG_hwpoison bit check.
                 */
                if (unlikely(folio_test_hwpoison(folio))) {
                        ret = VM_FAULT_HWPOISON_LARGE |
                                VM_FAULT_SET_HINDEX(hstate_index(h));
                        goto backout_unlocked;
                }

                /* Check for page in userfault range. */
                if (userfaultfd_minor(vma)) {
                        folio_unlock(folio);
                        folio_put(folio);
                        /* See comment in userfaultfd_missing() block above */
                        if (!hugetlb_pte_stable(h, mm, vmf->address, vmf->pte, vmf->orig_pte)) {
                                ret = 0;
                                goto out;
                        }
                        return hugetlb_handle_userfault(vmf, mapping,
                                                        VM_UFFD_MINOR);
                }
        }

        /*
         * If we are going to COW a private mapping later, we examine the
         * pending reservations for this page now. This will ensure that
         * any allocations necessary to record that reservation occur outside
         * the spinlock.
         */
        if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
                if (vma_needs_reservation(h, vma, vmf->address) < 0) {
                        ret = VM_FAULT_OOM;
                        goto backout_unlocked;
                }
                /* Just decrements count, does not deallocate */
                vma_end_reservation(h, vma, vmf->address);
        }

        vmf->ptl = huge_pte_lock(h, mm, vmf->pte);
        ret = 0;
        /* If pte changed from under us, retry */
        if (!pte_same(huge_ptep_get(mm, vmf->address, vmf->pte), vmf->orig_pte))
                goto backout;

        if (new_anon_folio)
                hugetlb_add_new_anon_rmap(folio, vma, vmf->address);
        else
                hugetlb_add_file_rmap(folio);
        new_pte = make_huge_pte(vma, folio, vma->vm_flags & VM_SHARED);
        /*
         * If this pte was previously wr-protected, keep it wr-protected even
         * if populated.
         */
        if (unlikely(pte_is_uffd_wp_marker(vmf->orig_pte)))
                new_pte = huge_pte_mkuffd(new_pte);
        set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h));

        hugetlb_count_add(pages_per_huge_page(h), mm);
        if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
                /*
                 * No need to keep file folios locked. See comment in
                 * hugetlb_fault().
                 */
                if (!new_anon_folio) {
                        folio_locked = false;
                        folio_unlock(folio);
                }
                /* Optimization, do the COW without a second fault */
                ret = hugetlb_wp(vmf);
        }

        spin_unlock(vmf->ptl);

        /*
         * Only set hugetlb_migratable in newly allocated pages.  Existing pages
         * found in the pagecache may not have hugetlb_migratable if they have
         * been isolated for migration.
         */
        if (new_folio)
                folio_set_hugetlb_migratable(folio);

        if (folio_locked)
                folio_unlock(folio);
out:
        hugetlb_vma_unlock_read(vma);

        /*
         * We must check to release the per-VMA lock. __vmf_anon_prepare() is
         * the only way ret can be set to VM_FAULT_RETRY.
         */
        if (unlikely(ret & VM_FAULT_RETRY))
                vma_end_read(vma);

        mutex_unlock(&hugetlb_fault_mutex_table[hash]);
        return ret;

backout:
        spin_unlock(vmf->ptl);
backout_unlocked:
        /* We only need to restore reservations for private mappings */
        if (new_anon_folio)
                restore_reserve_on_error(h, vma, vmf->address, folio);

        folio_unlock(folio);
        folio_put(folio);
        goto out;
}

#ifdef CONFIG_SMP
u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
{
        unsigned long key[2];
        u32 hash;

        key[0] = (unsigned long) mapping;
        key[1] = idx;

        hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);

        return hash & (num_fault_mutexes - 1);
}
#else
/*
 * For uniprocessor systems we always use a single mutex, so just
 * return 0 and avoid the hashing overhead.
 */
u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
{
        return 0;
}
#endif

vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                        unsigned long address, unsigned int flags)
{
        vm_fault_t ret;
        u32 hash;
        struct folio *folio = NULL;
        struct hstate *h = hstate_vma(vma);
        struct address_space *mapping;
        bool need_wait_lock = false;
        struct vm_fault vmf = {
                .vma = vma,
                .address = address & huge_page_mask(h),
                .real_address = address,
                .flags = flags,
                .pgoff = vma_hugecache_offset(h, vma,
                                address & huge_page_mask(h)),
                /* TODO: Track hugetlb faults using vm_fault */

                /*
                 * Some fields may not be initialized, be careful as it may
                 * be hard to debug if called functions make assumptions
                 */
        };

        /*
         * Serialize hugepage allocation and instantiation, so that we don't
         * get spurious allocation failures if two CPUs race to instantiate
         * the same page in the page cache.
         */
        mapping = vma->vm_file->f_mapping;
        hash = hugetlb_fault_mutex_hash(mapping, vmf.pgoff);
        mutex_lock(&hugetlb_fault_mutex_table[hash]);

        /*
         * Acquire vma lock before calling huge_pte_alloc and hold
         * until finished with vmf.pte.  This prevents huge_pmd_unshare from
         * being called elsewhere and making the vmf.pte no longer valid.
         */
        hugetlb_vma_lock_read(vma);
        vmf.pte = huge_pte_alloc(mm, vma, vmf.address, huge_page_size(h));
        if (!vmf.pte) {
                hugetlb_vma_unlock_read(vma);
                mutex_unlock(&hugetlb_fault_mutex_table[hash]);
                return VM_FAULT_OOM;
        }

        vmf.orig_pte = huge_ptep_get(mm, vmf.address, vmf.pte);
        if (huge_pte_none(vmf.orig_pte))
                /*
                 * hugetlb_no_page will drop vma lock and hugetlb fault
                 * mutex internally, which make us return immediately.
                 */
                return hugetlb_no_page(mapping, &vmf);

        if (pte_is_marker(vmf.orig_pte)) {
                const pte_marker marker =
                        softleaf_to_marker(softleaf_from_pte(vmf.orig_pte));

                if (marker & PTE_MARKER_POISONED) {
                        ret = VM_FAULT_HWPOISON_LARGE |
                                VM_FAULT_SET_HINDEX(hstate_index(h));
                        goto out_mutex;
                } else if (WARN_ON_ONCE(marker & PTE_MARKER_GUARD)) {
                        /* This isn't supported in hugetlb. */
                        ret = VM_FAULT_SIGSEGV;
                        goto out_mutex;
                }

                return hugetlb_no_page(mapping, &vmf);
        }

        ret = 0;

        /* Not present, either a migration or a hwpoisoned entry */
        if (!pte_present(vmf.orig_pte) && !huge_pte_none(vmf.orig_pte)) {
                const softleaf_t softleaf = softleaf_from_pte(vmf.orig_pte);

                if (softleaf_is_migration(softleaf)) {
                        /*
                         * Release the hugetlb fault lock now, but retain
                         * the vma lock, because it is needed to guard the
                         * huge_pte_lockptr() later in
                         * migration_entry_wait_huge(). The vma lock will
                         * be released there.
                         */
                        mutex_unlock(&hugetlb_fault_mutex_table[hash]);
                        migration_entry_wait_huge(vma, vmf.address, vmf.pte);
                        return 0;
                }
                if (softleaf_is_hwpoison(softleaf)) {
                        ret = VM_FAULT_HWPOISON_LARGE |
                            VM_FAULT_SET_HINDEX(hstate_index(h));
                }

                goto out_mutex;
        }

        /*
         * Protnone hugetlb PTEs with the uffd bit are used by
         * userfaultfd RWP for access tracking. Plain PROT_NONE (without the
         * marker) is not an RWP fault and is not expected on hugetlb (no
         * NUMA hinting), so let normal hugetlb fault handling proceed.
         */
        if (pte_protnone(vmf.orig_pte) && vma_is_accessible(vma) &&
            userfaultfd_rwp(vma) && huge_pte_uffd(vmf.orig_pte)) {
                spinlock_t *ptl;
                pte_t pte;

                /* Sync: drop hugetlb locks before blocking in handle_userfault() */
                if (!userfaultfd_rwp_async(vma))
                        return hugetlb_handle_userfault(&vmf, mapping, VM_UFFD_RWP);

                ptl = huge_pte_lock(h, mm, vmf.pte);
                pte = huge_ptep_get(mm, vmf.address, vmf.pte);
                if (pte_protnone(pte) && huge_pte_uffd(pte)) {
                        unsigned int shift = huge_page_shift(h);

                        pte = huge_pte_modify(pte, vma->vm_page_prot);
                        pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
                        /* huge_pte_modify() preserves _PAGE_UFFD; drop it on resolution */
                        pte = huge_pte_clear_uffd(pte);
                        pte = pte_mkyoung(pte);
                        /*
                         * Unlike do_uffd_rwp(), do not upgrade to writable
                         * here. Hugetlb lacks a can_change_huge_pte_writable()
                         * equivalent, so a write access will take a separate
                         * COW fault — acceptable for the rare private hugetlb
                         * case.
                         */
                        set_huge_pte_at(mm, vmf.address, vmf.pte, pte,
                                        huge_page_size(h));
                        update_mmu_cache(vma, vmf.address, vmf.pte);
                }
                spin_unlock(ptl);
                ret = 0;
                goto out_mutex;
        }

        /*
         * If we are going to COW/unshare the mapping later, we examine the
         * pending reservations for this page now. This will ensure that any
         * allocations necessary to record that reservation occur outside the
         * spinlock.
         */
        if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
            !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(vmf.orig_pte)) {
                if (vma_needs_reservation(h, vma, vmf.address) < 0) {
                        ret = VM_FAULT_OOM;
                        goto out_mutex;
                }
                /* Just decrements count, does not deallocate */
                vma_end_reservation(h, vma, vmf.address);
        }

        vmf.ptl = huge_pte_lock(h, mm, vmf.pte);

        /* Check for a racing update before calling hugetlb_wp() */
        if (unlikely(!pte_same(vmf.orig_pte, huge_ptep_get(mm, vmf.address, vmf.pte))))
                goto out_ptl;

        /* Handle userfault-wp first, before trying to lock more pages */
        if (userfaultfd_wp(vma) && huge_pte_uffd(huge_ptep_get(mm, vmf.address, vmf.pte)) &&
            (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) {
                if (!userfaultfd_wp_async(vma)) {
                        spin_unlock(vmf.ptl);
                        hugetlb_vma_unlock_read(vma);
                        mutex_unlock(&hugetlb_fault_mutex_table[hash]);
                        return handle_userfault(&vmf, VM_UFFD_WP);
                }

                vmf.orig_pte = huge_pte_clear_uffd(vmf.orig_pte);
                set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte,
                                huge_page_size(hstate_vma(vma)));
                /* Fallthrough to CoW */
        }

        if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
                if (!huge_pte_write(vmf.orig_pte)) {
                        /*
                         * Anonymous folios need to be lock since hugetlb_wp()
                         * checks whether we can re-use the folio exclusively
                         * for us in case we are the only user of it.
                         */
                        folio = page_folio(pte_page(vmf.orig_pte));
                        if (folio_test_anon(folio) && !folio_trylock(folio)) {
                                need_wait_lock = true;
                                goto out_ptl;
                        }
                        folio_get(folio);
                        ret = hugetlb_wp(&vmf);
                        if (folio_test_anon(folio))
                                folio_unlock(folio);
                        folio_put(folio);
                        goto out_ptl;
                } else if (likely(flags & FAULT_FLAG_WRITE)) {
                        vmf.orig_pte = huge_pte_mkdirty(vmf.orig_pte);
                }
        }
        vmf.orig_pte = pte_mkyoung(vmf.orig_pte);
        if (huge_ptep_set_access_flags(vma, vmf.address, vmf.pte, vmf.orig_pte,
                                                flags & FAULT_FLAG_WRITE))
                update_mmu_cache(vma, vmf.address, vmf.pte);
out_ptl:
        spin_unlock(vmf.ptl);
out_mutex:
        hugetlb_vma_unlock_read(vma);

        /*
         * We must check to release the per-VMA lock. __vmf_anon_prepare() in
         * hugetlb_wp() is the only way ret can be set to VM_FAULT_RETRY.
         */
        if (unlikely(ret & VM_FAULT_RETRY))
                vma_end_read(vma);

        mutex_unlock(&hugetlb_fault_mutex_table[hash]);
        /*
         * hugetlb_wp drops all the locks, but the folio lock, before trying to
         * unmap the folio from other processes. During that window, if another
         * process mapping that folio faults in, it will take the mutex and then
         * it will wait on folio_lock, causing an ABBA deadlock.
         * Use trylock instead and bail out if we fail.
         *
         * Ideally, we should hold a refcount on the folio we wait for, but we do
         * not want to use the folio after it becomes unlocked, but rather just
         * wait for it to become unlocked, so hopefully next fault successes on
         * the trylock.
         */
        if (need_wait_lock)
                folio_wait_locked(folio);
        return ret;
}

#ifdef CONFIG_USERFAULTFD
/*
 * Can probably be eliminated, but still used by hugetlb_mfill_atomic_pte().
 */
static struct folio *alloc_hugetlb_folio_vma(struct hstate *h,
                struct vm_area_struct *vma, unsigned long address)
{
        struct mempolicy *mpol;
        nodemask_t *nodemask;
        struct folio *folio;
        gfp_t gfp_mask;
        int node;

        gfp_mask = htlb_alloc_mask(h);
        node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
        /*
         * This is used to allocate a temporary hugetlb to hold the copied
         * content, which will then be copied again to the final hugetlb
         * consuming a reservation. Set the alloc_fallback to false to indicate
         * that breaking the per-node hugetlb pool is not allowed in this case.
         */
        folio = alloc_hugetlb_folio_nodemask(h, node, nodemask, gfp_mask, false);
        mpol_cond_put(mpol);

        return folio;
}

/*
 * Used by userfaultfd UFFDIO_* ioctls. Based on userfaultfd's mfill_atomic_pte
 * with modifications for hugetlb pages.
 */
int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
                             struct vm_area_struct *dst_vma,
                             unsigned long dst_addr,
                             unsigned long src_addr,
                             uffd_flags_t flags,
                             struct folio **foliop)
{
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        bool is_continue = uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE);
        bool wp_enabled = (flags & MFILL_ATOMIC_WP);
        struct hstate *h = hstate_vma(dst_vma);
        struct address_space *mapping = dst_vma->vm_file->f_mapping;
        pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr);
        unsigned long size = huge_page_size(h);
        int vm_shared = dst_vma->vm_flags & VM_SHARED;
        pte_t _dst_pte;
        spinlock_t *ptl;
        int ret = -ENOMEM;
        struct folio *folio;
        bool folio_in_pagecache = false;
        pte_t dst_ptep;

        if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON)) {
                ptl = huge_pte_lock(h, dst_mm, dst_pte);

                /* Don't overwrite any existing PTEs (even markers) */
                if (!huge_pte_none(huge_ptep_get(dst_mm, dst_addr, dst_pte))) {
                        spin_unlock(ptl);
                        return -EEXIST;
                }

                _dst_pte = make_pte_marker(PTE_MARKER_POISONED);
                set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size);

                /* No need to invalidate - it was non-present before */
                update_mmu_cache(dst_vma, dst_addr, dst_pte);

                spin_unlock(ptl);
                return 0;
        }

        if (is_continue) {
                ret = -EFAULT;
                folio = filemap_lock_hugetlb_folio(h, mapping, idx);
                if (IS_ERR(folio))
                        goto out;
                folio_in_pagecache = true;
        } else if (!*foliop) {
                /* If a folio already exists, then it's UFFDIO_COPY for
                 * a non-missing case. Return -EEXIST.
                 */
                if (vm_shared &&
                    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
                        ret = -EEXIST;
                        goto out;
                }

                folio = alloc_hugetlb_folio(dst_vma, dst_addr, false);
                if (IS_ERR(folio)) {
                        pte_t *actual_pte = hugetlb_walk(dst_vma, dst_addr, PMD_SIZE);
                        if (actual_pte) {
                                ret = -EEXIST;
                                goto out;
                        }
                        ret = -ENOMEM;
                        goto out;
                }

                ret = copy_folio_from_user(folio, (const void __user *) src_addr,
                                           false);

                /* fallback to copy_from_user outside mmap_lock */
                if (unlikely(ret)) {
                        ret = -ENOENT;
                        /* Free the allocated folio which may have
                         * consumed a reservation.
                         */
                        restore_reserve_on_error(h, dst_vma, dst_addr, folio);
                        folio_put(folio);

                        /* Allocate a temporary folio to hold the copied
                         * contents.
                         */
                        folio = alloc_hugetlb_folio_vma(h, dst_vma, dst_addr);
                        if (!folio) {
                                ret = -ENOMEM;
                                goto out;
                        }
                        *foliop = folio;
                        /* Set the outparam foliop and return to the caller to
                         * copy the contents outside the lock. Don't free the
                         * folio.
                         */
                        goto out;
                }
        } else {
                if (vm_shared &&
                    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
                        folio_put(*foliop);
                        ret = -EEXIST;
                        *foliop = NULL;
                        goto out;
                }

                folio = alloc_hugetlb_folio(dst_vma, dst_addr, false);
                if (IS_ERR(folio)) {
                        folio_put(*foliop);
                        ret = -ENOMEM;
                        *foliop = NULL;
                        goto out;
                }
                ret = copy_user_large_folio(folio, *foliop, dst_addr, dst_vma);
                folio_put(*foliop);
                *foliop = NULL;
                if (ret) {
                        restore_reserve_on_error(h, dst_vma, dst_addr, folio);
                        folio_put(folio);
                        goto out;
                }
        }

        /*
         * If we just allocated a new page, we need a memory barrier to ensure
         * that preceding stores to the page become visible before the
         * set_pte_at() write. The memory barrier inside __folio_mark_uptodate
         * is what we need.
         *
         * In the case where we have not allocated a new page (is_continue),
         * the page must already be uptodate. UFFDIO_CONTINUE already includes
         * an earlier smp_wmb() to ensure that prior stores will be visible
         * before the set_pte_at() write.
         */
        if (!is_continue)
                __folio_mark_uptodate(folio);
        else
                WARN_ON_ONCE(!folio_test_uptodate(folio));

        /* Add shared, newly allocated pages to the page cache. */
        if (vm_shared && !is_continue) {
                ret = -EFAULT;
                if (idx >= (i_size_read(mapping->host) >> huge_page_shift(h)))
                        goto out_release_nounlock;

                /*
                 * Serialization between remove_inode_hugepages() and
                 * hugetlb_add_to_page_cache() below happens through the
                 * hugetlb_fault_mutex_table that here must be hold by
                 * the caller.
                 */
                ret = hugetlb_add_to_page_cache(folio, mapping, idx);
                if (ret)
                        goto out_release_nounlock;
                folio_in_pagecache = true;
        }

        ptl = huge_pte_lock(h, dst_mm, dst_pte);

        ret = -EIO;
        if (folio_test_hwpoison(folio))
                goto out_release_unlock;

        ret = -EEXIST;

        dst_ptep = huge_ptep_get(dst_mm, dst_addr, dst_pte);
        /*
         * See comment about UFFD marker overwriting in
         * mfill_atomic_install_pte().
         */
        if (!huge_pte_none(dst_ptep) && !pte_is_uffd_marker(dst_ptep))
                goto out_release_unlock;

        if (folio_in_pagecache)
                hugetlb_add_file_rmap(folio);
        else
                hugetlb_add_new_anon_rmap(folio, dst_vma, dst_addr);

        /*
         * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY
         * with wp flag set, don't set pte write bit.
         */
        _dst_pte = make_huge_pte(dst_vma, folio,
                                 !wp_enabled && !(is_continue && !vm_shared));
        /*
         * Always mark UFFDIO_COPY page dirty; note that this may not be
         * extremely important for hugetlbfs for now since swapping is not
         * supported, but we should still be clear in that this page cannot be
         * thrown away at will, even if write bit not set.
         */
        _dst_pte = huge_pte_mkdirty(_dst_pte);
        _dst_pte = pte_mkyoung(_dst_pte);

        if (wp_enabled)
                _dst_pte = huge_pte_mkuffd(_dst_pte);

        set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size);

        hugetlb_count_add(pages_per_huge_page(h), dst_mm);

        /* No need to invalidate - it was non-present before */
        update_mmu_cache(dst_vma, dst_addr, dst_pte);

        spin_unlock(ptl);
        if (!is_continue)
                folio_set_hugetlb_migratable(folio);
        if (vm_shared || is_continue)
                folio_unlock(folio);
        ret = 0;
out:
        return ret;
out_release_unlock:
        spin_unlock(ptl);
        if (vm_shared || is_continue)
                folio_unlock(folio);
out_release_nounlock:
        if (!folio_in_pagecache)
                restore_reserve_on_error(h, dst_vma, dst_addr, folio);
        folio_put(folio);
        goto out;
}
#endif /* CONFIG_USERFAULTFD */

long hugetlb_change_protection(struct vm_area_struct *vma,
                unsigned long address, unsigned long end,
                pgprot_t newprot, unsigned long cp_flags)
{
        struct mm_struct *mm = vma->vm_mm;
        unsigned long start = address;
        pte_t *ptep;
        pte_t pte;
        struct hstate *h = hstate_vma(vma);
        long pages = 0, psize = huge_page_size(h);
        struct mmu_notifier_range range;
        unsigned long last_addr_mask;
        bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
        bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
        bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP;
        bool uffd_rwp_resolve = cp_flags & MM_CP_UFFD_RWP_RESOLVE;
        struct mmu_gather tlb;

        /*
         * In the case of shared PMDs, the area to flush could be beyond
         * start/end.  Set range.start/range.end to cover the maximum possible
         * range if PMD sharing is possible.
         */
        mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
                                0, mm, start, end);
        adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);

        BUG_ON(address >= end);
        flush_cache_range(vma, range.start, range.end);
        tlb_gather_mmu_vma(&tlb, vma);

        mmu_notifier_invalidate_range_start(&range);
        hugetlb_vma_lock_write(vma);
        i_mmap_lock_write(vma->vm_file->f_mapping);
        last_addr_mask = hugetlb_mask_last_page(h);
        for (; address < end; address += psize) {
                softleaf_t entry;
                spinlock_t *ptl;

                ptep = hugetlb_walk(vma, address, psize);
                if (!ptep) {
                        /*
                         * uffd_wp installs a pte marker on the unpopulated
                         * entry; uffd_rwp does not install markers so the
                         * allocation is unnecessary for it.
                         */
                        if (!uffd_wp) {
                                address |= last_addr_mask;
                                continue;
                        }
                        /*
                         * Userfaultfd wr-protect requires pgtable
                         * pre-allocations to install pte markers.
                         */
                        ptep = huge_pte_alloc(mm, vma, address, psize);
                        if (!ptep) {
                                pages = -ENOMEM;
                                break;
                        }
                }
                ptl = huge_pte_lock(h, mm, ptep);
                if (huge_pmd_unshare(&tlb, vma, address, ptep)) {
                        /*
                         * When uffd-wp is enabled on the vma, unshare
                         * shouldn't happen at all.  Warn about it if it
                         * happened due to some reason.
                         */
                        WARN_ON_ONCE(uffd_wp || uffd_wp_resolve ||
                                     uffd_rwp || uffd_rwp_resolve);
                        pages++;
                        spin_unlock(ptl);
                        address |= last_addr_mask;
                        continue;
                }
                pte = huge_ptep_get(mm, address, ptep);
                if (huge_pte_none(pte)) {
                        if (unlikely(uffd_wp))
                                /* Safe to modify directly (none->non-present). */
                                set_huge_pte_at(mm, address, ptep,
                                                make_pte_marker(PTE_MARKER_UFFD_WP),
                                                psize);
                        goto next;
                }

                entry = softleaf_from_pte(pte);
                if (unlikely(softleaf_is_hwpoison(entry))) {
                        /* Nothing to do. */
                } else if (unlikely(softleaf_is_migration(entry))) {
                        struct folio *folio = softleaf_to_folio(entry);
                        pte_t newpte = pte;

                        if (softleaf_is_migration_write(entry)) {
                                if (folio_test_anon(folio))
                                        entry = make_readable_exclusive_migration_entry(
                                                                swp_offset(entry));
                                else
                                        entry = make_readable_migration_entry(
                                                                swp_offset(entry));
                                newpte = swp_entry_to_pte(entry);
                                pages++;
                        }

                        if (uffd_wp || uffd_rwp)
                                newpte = pte_swp_mkuffd(newpte);
                        else if (uffd_wp_resolve || uffd_rwp_resolve)
                                newpte = pte_swp_clear_uffd(newpte);
                        if (!pte_same(pte, newpte))
                                set_huge_pte_at(mm, address, ptep, newpte, psize);
                } else if (unlikely(pte_is_marker(pte))) {
                        /*
                         * Do nothing on a poison marker; page is
                         * corrupted, permissions do not apply. Here
                         * pte_marker_uffd_wp()==true implies !poison
                         * because they're mutual exclusive.
                         */
                        if (pte_is_uffd_wp_marker(pte) &&
                            (uffd_wp_resolve || uffd_rwp_resolve))
                                /* Safe to modify directly (non-present->none). */
                                huge_pte_clear(mm, address, ptep, psize);
                } else {
                        pte_t old_pte;
                        unsigned int shift = huge_page_shift(hstate_vma(vma));

                        /* Already protnone with uffd bit set? Nothing to do. */
                        if (uffd_rwp && pte_protnone(pte) && huge_pte_uffd(pte))
                                goto next;

                        old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
                        pte = huge_pte_modify(old_pte, newprot);
                        pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
                        if (uffd_wp || uffd_rwp)
                                pte = huge_pte_mkuffd(pte);
                        else if (uffd_wp_resolve || uffd_rwp_resolve)
                                pte = huge_pte_clear_uffd(pte);

                        /* Preserve RWP protection across mprotect() */
                        if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) {
                                pte = huge_pte_modify(pte, PAGE_NONE);
                                pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
                        }

                        huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
                        pages++;
                        tlb_remove_huge_tlb_entry(h, &tlb, ptep, address);
                }

next:
                spin_unlock(ptl);
                cond_resched();
        }

        tlb_flush_mmu_tlbonly(&tlb);
        huge_pmd_unshare_flush(&tlb, vma);
        /*
         * No need to call mmu_notifier_arch_invalidate_secondary_tlbs() we are
         * downgrading page table protection not changing it to point to a new
         * page.
         *
         * See Documentation/mm/mmu_notifier.rst
         */
        i_mmap_unlock_write(vma->vm_file->f_mapping);
        hugetlb_vma_unlock_write(vma);
        mmu_notifier_invalidate_range_end(&range);
        tlb_finish_mmu(&tlb);

        return pages > 0 ? (pages << h->order) : pages;
}

/*
 * Update the reservation map for the range [from, to].
 *
 * Returns the number of entries that would be added to the reservation map
 * associated with the range [from, to].  This number is greater or equal to
 * zero. -EINVAL or -ENOMEM is returned in case of any errors.
 */

long hugetlb_reserve_pages(struct inode *inode,
                long from, long to,
                struct vm_area_struct *vma,
                vma_flags_t vma_flags)
{
        long chg = -1, add = -1, spool_resv, gbl_resv;
        struct hstate *h = hstate_inode(inode);
        struct hugepage_subpool *spool = subpool_inode(inode);
        struct resv_map *resv_map;
        struct hugetlb_cgroup *h_cg = NULL;
        long gbl_reserve, regions_needed = 0;
        int err;

        /* This should never happen */
        if (from > to) {
                VM_WARN(1, "%s called with a negative range\n", __func__);
                return -EINVAL;
        }

        /*
         * vma specific semaphore used for pmd sharing and fault/truncation
         * synchronization
         */
        hugetlb_vma_lock_alloc(vma);

        /*
         * Only apply hugepage reservation if asked. At fault time, an
         * attempt will be made for VM_NORESERVE to allocate a page
         * without using reserves
         */
        if (vma_flags_test(&vma_flags, VMA_NORESERVE_BIT))
                return 0;

        /*
         * Shared mappings base their reservation on the number of pages that
         * are already allocated on behalf of the file. Private mappings need
         * to reserve the full area even if read-only as mprotect() may be
         * called to make the mapping read-write. Assume !vma is a shm mapping
         */
        if (!vma || vma_test(vma, VMA_MAYSHARE_BIT)) {
                /*
                 * resv_map can not be NULL as hugetlb_reserve_pages is only
                 * called for inodes for which resv_maps were created (see
                 * hugetlbfs_get_inode).
                 */
                resv_map = inode_resv_map(inode);

                chg = region_chg(resv_map, from, to, &regions_needed);
        } else {
                /* Private mapping. */
                resv_map = resv_map_alloc();
                if (!resv_map) {
                        err = -ENOMEM;
                        goto out_err;
                }

                chg = to - from;

                set_vma_resv_map(vma, resv_map);
                set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
        }

        if (chg < 0) {
                /* region_chg() above can return -ENOMEM */
                err = (chg == -ENOMEM) ? -ENOMEM : -EINVAL;
                goto out_err;
        }

        err = hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h),
                                chg * pages_per_huge_page(h), &h_cg);
        if (err < 0)
                goto out_err;

        if (vma && !vma_test(vma, VMA_MAYSHARE_BIT) && h_cg) {
                /* For private mappings, the hugetlb_cgroup uncharge info hangs
                 * of the resv_map.
                 */
                resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h);
        }

        /*
         * There must be enough pages in the subpool for the mapping. If
         * the subpool has a minimum size, there may be some global
         * reservations already in place (gbl_reserve).
         */
        gbl_reserve = hugepage_subpool_get_pages(spool, chg);
        if (gbl_reserve < 0) {
                err = gbl_reserve;
                goto out_uncharge_cgroup;
        }

        /*
         * Check enough hugepages are available for the reservation.
         * Hand the pages back to the subpool if there are not
         */
        err = hugetlb_acct_memory(h, gbl_reserve);
        if (err < 0)
                goto out_put_pages;

        /*
         * Account for the reservations made. Shared mappings record regions
         * that have reservations as they are shared by multiple VMAs.
         * When the last VMA disappears, the region map says how much
         * the reservation was and the page cache tells how much of
         * the reservation was consumed. Private mappings are per-VMA and
         * only the consumed reservations are tracked. When the VMA
         * disappears, the original reservation is the VMA size and the
         * consumed reservations are stored in the map. Hence, nothing
         * else has to be done for private mappings here
         */
        if (!vma || vma_test(vma, VMA_MAYSHARE_BIT)) {
                add = region_add(resv_map, from, to, regions_needed, h, h_cg);

                if (unlikely(add < 0)) {
                        hugetlb_acct_memory(h, -gbl_reserve);
                        err = add;
                        goto out_put_pages;
                } else if (unlikely(chg > add)) {
                        /*
                         * pages in this range were added to the reserve
                         * map between region_chg and region_add.  This
                         * indicates a race with alloc_hugetlb_folio.  Adjust
                         * the subpool and reserve counts modified above
                         * based on the difference.
                         */
                        long rsv_adjust;

                        /*
                         * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the
                         * reference to h_cg->css. See comment below for detail.
                         */
                        hugetlb_cgroup_uncharge_cgroup_rsvd(
                                hstate_index(h),
                                (chg - add) * pages_per_huge_page(h), h_cg);

                        rsv_adjust = hugepage_subpool_put_pages(spool,
                                                                chg - add);
                        hugetlb_acct_memory(h, -rsv_adjust);
                } else if (h_cg) {
                        /*
                         * The file_regions will hold their own reference to
                         * h_cg->css. So we should release the reference held
                         * via hugetlb_cgroup_charge_cgroup_rsvd() when we are
                         * done.
                         */
                        hugetlb_cgroup_put_rsvd_cgroup(h_cg);
                }
        }
        return chg;

out_put_pages:
        spool_resv = chg - gbl_reserve;
        if (spool_resv) {
                /* put sub pool's reservation back, chg - gbl_reserve */
                gbl_resv = hugepage_subpool_put_pages(spool, spool_resv);
                /*
                 * subpool's reserved pages can not be put back due to race,
                 * return to hstate.
                 */
                hugetlb_acct_memory(h, -gbl_resv);
        }
        /* Restore used_hpages for pages that failed global reservation */
        if (gbl_reserve && spool) {
                unsigned long flags;

                spin_lock_irqsave(&spool->lock, flags);
                if (spool->max_hpages != -1)
                        spool->used_hpages -= gbl_reserve;
                unlock_or_release_subpool(spool, flags);
        }
out_uncharge_cgroup:
        hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
                                            chg * pages_per_huge_page(h), h_cg);
out_err:
        hugetlb_vma_lock_free(vma);
        if (!vma || vma_test(vma, VMA_MAYSHARE_BIT))
                /* Only call region_abort if the region_chg succeeded but the
                 * region_add failed or didn't run.
                 */
                if (chg >= 0 && add < 0)
                        region_abort(resv_map, from, to, regions_needed);
        if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
                kref_put(&resv_map->refs, resv_map_release);
                set_vma_resv_map(vma, NULL);
        }
        return err;
}

long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
                                                                long freed)
{
        struct hstate *h = hstate_inode(inode);
        struct resv_map *resv_map = inode_resv_map(inode);
        long chg = 0;
        struct hugepage_subpool *spool = subpool_inode(inode);
        long gbl_reserve;

        /*
         * Since this routine can be called in the evict inode path for all
         * hugetlbfs inodes, resv_map could be NULL.
         */
        if (resv_map) {
                chg = region_del(resv_map, start, end);
                /*
                 * region_del() can fail in the rare case where a region
                 * must be split and another region descriptor can not be
                 * allocated.  If end == LONG_MAX, it will not fail.
                 */
                if (chg < 0)
                        return chg;
        }

        spin_lock(&inode->i_lock);
        inode->i_blocks -= (blocks_per_huge_page(h) * freed);
        spin_unlock(&inode->i_lock);

        /*
         * If the subpool has a minimum size, the number of global
         * reservations to be released may be adjusted.
         *
         * Note that !resv_map implies freed == 0. So (chg - freed)
         * won't go negative.
         */
        gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
        hugetlb_acct_memory(h, -gbl_reserve);

        return 0;
}

#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
static unsigned long page_table_shareable(struct vm_area_struct *svma,
                                struct vm_area_struct *vma,
                                unsigned long addr, pgoff_t idx)
{
        unsigned long saddr = ((idx - vma_start_pgoff(svma)) << PAGE_SHIFT) +
                                svma->vm_start;
        unsigned long sbase = saddr & PUD_MASK;
        unsigned long s_end = sbase + PUD_SIZE;

        /* Allow segments to share if only one is marked locked */
        vm_flags_t vm_flags = vma->vm_flags & ~VM_LOCKED_MASK;
        vm_flags_t svm_flags = svma->vm_flags & ~VM_LOCKED_MASK;

        /*
         * match the virtual addresses, permission and the alignment of the
         * page table page.
         *
         * Also, vma_lock (vm_private_data) is required for sharing.
         */
        if (pmd_index(addr) != pmd_index(saddr) ||
            vm_flags != svm_flags ||
            !range_in_vma(svma, sbase, s_end) ||
            !svma->vm_private_data)
                return 0;

        return saddr;
}

bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
{
        unsigned long start = addr & PUD_MASK;
        unsigned long end = start + PUD_SIZE;

#ifdef CONFIG_USERFAULTFD
        if (uffd_disable_huge_pmd_share(vma))
                return false;
#endif
        /*
         * check on proper vm_flags and page table alignment
         */
        if (!(vma->vm_flags & VM_MAYSHARE))
                return false;
        if (!vma->vm_private_data)        /* vma lock required for sharing */
                return false;
        if (!range_in_vma(vma, start, end))
                return false;
        return true;
}

/*
 * Determine if start,end range within vma could be mapped by shared pmd.
 * If yes, adjust start and end to cover range associated with possible
 * shared pmd mappings.
 */
void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
                                unsigned long *start, unsigned long *end)
{
        unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
                v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);

        /*
         * vma needs to span at least one aligned PUD size, and the range
         * must be at least partially within in.
         */
        if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
                (*end <= v_start) || (*start >= v_end))
                return;

        /* Extend the range to be PUD aligned for a worst case scenario */
        if (*start > v_start)
                *start = ALIGN_DOWN(*start, PUD_SIZE);

        if (*end < v_end)
                *end = ALIGN(*end, PUD_SIZE);
}

/*
 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
 * and returns the corresponding pte. While this is not necessary for the
 * !shared pmd case because we can allocate the pmd later as well, it makes the
 * code much cleaner. pmd allocation is essential for the shared case because
 * pud has to be populated inside the same i_mmap_rwsem section - otherwise
 * racing tasks could either miss the sharing (see huge_pte_offset) or select a
 * bad pmd for sharing.
 */
pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
                      unsigned long addr, pud_t *pud)
{
        struct address_space *mapping = vma->vm_file->f_mapping;
        const pgoff_t idx = linear_page_index(vma, addr);
        struct vm_area_struct *svma;
        unsigned long saddr;
        pte_t *spte = NULL;
        pte_t *pte;

        i_mmap_lock_read(mapping);
        mapping_rmap_tree_foreach(svma, mapping, idx, idx) {
                if (svma == vma)
                        continue;

                saddr = page_table_shareable(svma, vma, addr, idx);
                if (saddr) {
                        spte = hugetlb_walk(svma, saddr,
                                            vma_mmu_pagesize(svma));
                        if (spte) {
                                ptdesc_pmd_pts_inc(virt_to_ptdesc(spte));
                                break;
                        }
                }
        }

        if (!spte)
                goto out;

        spin_lock(&mm->page_table_lock);
        if (pud_none(*pud)) {
                pud_populate(mm, pud,
                                (pmd_t *)((unsigned long)spte & PAGE_MASK));
                mm_inc_nr_pmds(mm);
        } else {
                ptdesc_pmd_pts_dec(virt_to_ptdesc(spte));
        }
        spin_unlock(&mm->page_table_lock);
out:
        pte = (pte_t *)pmd_alloc(mm, pud, addr);
        i_mmap_unlock_read(mapping);
        return pte;
}

static int __huge_pmd_unshare(struct mmu_gather *tlb,
                struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
                bool check_locks)
{
        unsigned long sz = huge_page_size(hstate_vma(vma));
        struct mm_struct *mm = vma->vm_mm;
        pgd_t *pgd = pgd_offset(mm, addr);
        p4d_t *p4d = p4d_offset(pgd, addr);
        pud_t *pud = pud_offset(p4d, addr);

        if (sz != PMD_SIZE)
                return 0;
        if (!ptdesc_pmd_is_shared(virt_to_ptdesc(ptep)))
                return 0;
        i_mmap_assert_write_locked(vma->vm_file->f_mapping);
        if (check_locks)
                hugetlb_vma_assert_locked(vma);
        pud_clear(pud);

        tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr);

        mm_dec_nr_pmds(mm);
        return 1;
}

/**
 * huge_pmd_unshare - Unmap a pmd table if it is shared by multiple users
 * @tlb: the current mmu_gather.
 * @vma: the vma covering the pmd table.
 * @addr: the address we are trying to unshare.
 * @ptep: pointer into the (pmd) page table.
 *
 * Called with the page table lock held, the i_mmap_rwsem held in write mode
 * and the hugetlb vma lock held in write mode.
 *
 * Note: The caller must call huge_pmd_unshare_flush() before dropping the
 * i_mmap_rwsem.
 *
 * Returns: 1 if it was a shared PMD table and it got unmapped, or 0 if it
 *            was not a shared PMD table.
 */
int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
                unsigned long addr, pte_t *ptep)
{
        return __huge_pmd_unshare(tlb, vma, addr, ptep, /*check_locks=*/true);
}

/*
 * huge_pmd_unshare_flush - Complete a sequence of huge_pmd_unshare() calls
 * @tlb: the current mmu_gather.
 * @vma: the vma covering the pmd table.
 *
 * Perform necessary TLB flushes or IPI broadcasts to synchronize PMD table
 * unsharing with concurrent page table walkers.
 *
 * This function must be called after a sequence of huge_pmd_unshare()
 * calls while still holding the i_mmap_rwsem.
 */
void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma)
{
        /*
         * We must synchronize page table unsharing such that nobody will
         * try reusing a previously-shared page table while it might still
         * be in use by previous sharers (TLB, GUP_fast).
         */
        i_mmap_assert_write_locked(vma->vm_file->f_mapping);

        tlb_flush_unshared_tables(tlb);
}

#else /* !CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */

pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
                      unsigned long addr, pud_t *pud)
{
        return NULL;
}

static int __huge_pmd_unshare(struct mmu_gather *tlb,
                struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
                bool check_locks)
{
        return 0;
}

int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
                unsigned long addr, pte_t *ptep)
{
        return 0;
}

void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma)
{
}

void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
                                unsigned long *start, unsigned long *end)
{
}

bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
{
        return false;
}
#endif /* CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */

#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
                        unsigned long addr, unsigned long sz)
{
        pgd_t *pgd;
        p4d_t *p4d;
        pud_t *pud;
        pte_t *pte = NULL;

        pgd = pgd_offset(mm, addr);
        p4d = p4d_alloc(mm, pgd, addr);
        if (!p4d)
                return NULL;
        pud = pud_alloc(mm, p4d, addr);
        if (pud) {
                if (sz == PUD_SIZE) {
                        pte = (pte_t *)pud;
                } else {
                        BUG_ON(sz != PMD_SIZE);
                        if (want_pmd_share(vma, addr) && pud_none(*pud))
                                pte = huge_pmd_share(mm, vma, addr, pud);
                        else
                                pte = (pte_t *)pmd_alloc(mm, pud, addr);
                }
        }

        if (pte) {
                pte_t pteval = ptep_get_lockless(pte);

                BUG_ON(pte_present(pteval) && !pte_huge(pteval));
        }

        return pte;
}

/*
 * huge_pte_offset() - Walk the page table to resolve the hugepage
 * entry at address @addr
 *
 * Return: Pointer to page table entry (PUD or PMD) for
 * address @addr, or NULL if a !p*d_present() entry is encountered and the
 * size @sz doesn't match the hugepage size at this level of the page
 * table.
 */
pte_t *huge_pte_offset(struct mm_struct *mm,
                       unsigned long addr, unsigned long sz)
{
        pgd_t *pgd;
        p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;

        pgd = pgd_offset(mm, addr);
        if (!pgd_present(*pgd))
                return NULL;
        p4d = p4d_offset(pgd, addr);
        if (!p4d_present(*p4d))
                return NULL;

        pud = pud_offset(p4d, addr);
        if (sz == PUD_SIZE)
                /* must be pud huge, non-present or none */
                return (pte_t *)pud;
        if (!pud_present(*pud))
                return NULL;
        /* must have a valid entry and size to go further */

        pmd = pmd_offset(pud, addr);
        /* must be pmd huge, non-present or none */
        return (pte_t *)pmd;
}

/*
 * Return a mask that can be used to update an address to the last huge
 * page in a page table page mapping size.  Used to skip non-present
 * page table entries when linearly scanning address ranges.  Architectures
 * with unique huge page to page table relationships can define their own
 * version of this routine.
 */
unsigned long hugetlb_mask_last_page(struct hstate *h)
{
        unsigned long hp_size = huge_page_size(h);

        if (hp_size == PUD_SIZE)
                return P4D_SIZE - PUD_SIZE;
        else if (hp_size == PMD_SIZE)
                return PUD_SIZE - PMD_SIZE;
        else
                return 0UL;
}

#else

/* See description above.  Architectures can provide their own version. */
__weak unsigned long hugetlb_mask_last_page(struct hstate *h)
{
#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
        if (huge_page_size(h) == PMD_SIZE)
                return PUD_SIZE - PMD_SIZE;
#endif
        return 0UL;
}

#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */

/**
 * folio_isolate_hugetlb - try to isolate an allocated hugetlb folio
 * @folio: the folio to isolate
 * @list: the list to add the folio to on success
 *
 * Isolate an allocated (refcount > 0) hugetlb folio, marking it as
 * isolated/non-migratable, and moving it from the active list to the
 * given list.
 *
 * Isolation will fail if @folio is not an allocated hugetlb folio, or if
 * it is already isolated/non-migratable.
 *
 * On success, an additional folio reference is taken that must be dropped
 * using folio_putback_hugetlb() to undo the isolation.
 *
 * Return: True if isolation worked, otherwise False.
 */
bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list)
{
        bool ret = true;

        spin_lock_irq(&hugetlb_lock);
        if (!folio_test_hugetlb(folio) ||
            !folio_test_hugetlb_migratable(folio) ||
            !folio_try_get(folio)) {
                ret = false;
                goto unlock;
        }
        folio_clear_hugetlb_migratable(folio);
        list_move_tail(&folio->lru, list);
unlock:
        spin_unlock_irq(&hugetlb_lock);
        return ret;
}

int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison)
{
        int ret = 0;

        *hugetlb = false;
        spin_lock_irq(&hugetlb_lock);
        if (folio_test_hugetlb(folio)) {
                *hugetlb = true;
                if (folio_test_hugetlb_freed(folio))
                        ret = 0;
                else if (folio_test_hugetlb_migratable(folio) || unpoison)
                        ret = folio_try_get(folio);
                else
                        ret = -EBUSY;
        }
        spin_unlock_irq(&hugetlb_lock);
        return ret;
}

/**
 * folio_putback_hugetlb - unisolate a hugetlb folio
 * @folio: the isolated hugetlb folio
 *
 * Putback/un-isolate the hugetlb folio that was previous isolated using
 * folio_isolate_hugetlb(): marking it non-isolated/migratable and putting it
 * back onto the active list.
 *
 * Will drop the additional folio reference obtained through
 * folio_isolate_hugetlb().
 */
void folio_putback_hugetlb(struct folio *folio)
{
        spin_lock_irq(&hugetlb_lock);
        folio_set_hugetlb_migratable(folio);
        list_move_tail(&folio->lru, &(folio_hstate(folio))->hugepage_activelist);
        spin_unlock_irq(&hugetlb_lock);
        folio_put(folio);
}

void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio,
                        enum migrate_reason reason)
{
        struct hstate *h = folio_hstate(old_folio);

        hugetlb_cgroup_migrate(old_folio, new_folio);
        folio_set_owner_migrate_reason(new_folio, reason);

        /*
         * transfer temporary state of the new hugetlb folio. This is
         * reverse to other transitions because the newpage is going to
         * be final while the old one will be freed so it takes over
         * the temporary status.
         *
         * Also note that we have to transfer the per-node surplus state
         * here as well otherwise the global surplus count will not match
         * the per-node's.
         */
        if (folio_test_hugetlb_temporary(new_folio)) {
                int old_nid = folio_nid(old_folio);
                int new_nid = folio_nid(new_folio);

                folio_set_hugetlb_temporary(old_folio);
                folio_clear_hugetlb_temporary(new_folio);


                /*
                 * There is no need to transfer the per-node surplus state
                 * when we do not cross the node.
                 */
                if (new_nid == old_nid)
                        return;
                spin_lock_irq(&hugetlb_lock);
                if (h->surplus_huge_pages_node[old_nid]) {
                        h->surplus_huge_pages_node[old_nid]--;
                        h->surplus_huge_pages_node[new_nid]++;
                }
                spin_unlock_irq(&hugetlb_lock);
        }

        /*
         * Our old folio is isolated and has "migratable" cleared until it
         * is putback. As migration succeeded, set the new folio "migratable"
         * and add it to the active list.
         */
        spin_lock_irq(&hugetlb_lock);
        folio_set_hugetlb_migratable(new_folio);
        list_move_tail(&new_folio->lru, &(folio_hstate(new_folio))->hugepage_activelist);
        spin_unlock_irq(&hugetlb_lock);
}

/*
 * If @take_locks is false, the caller must ensure that no concurrent page table
 * access can happen (except for gup_fast() and hardware page walks).
 * If @take_locks is true, we take the hugetlb VMA lock (to lock out things like
 * concurrent page fault handling) and the file rmap lock.
 */
static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
                                   unsigned long start,
                                   unsigned long end,
                                   bool take_locks)
{
        struct hstate *h = hstate_vma(vma);
        unsigned long sz = huge_page_size(h);
        struct mm_struct *mm = vma->vm_mm;
        struct mmu_notifier_range range;
        struct mmu_gather tlb;
        unsigned long address;
        spinlock_t *ptl;
        pte_t *ptep;

        if (!(vma->vm_flags & VM_MAYSHARE))
                return;

        if (start >= end)
                return;

        flush_cache_range(vma, start, end);
        tlb_gather_mmu_vma(&tlb, vma);

        /*
         * No need to call adjust_range_if_pmd_sharing_possible(), because
         * we have already done the PUD_SIZE alignment.
         */
        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
                                start, end);
        mmu_notifier_invalidate_range_start(&range);
        if (take_locks) {
                hugetlb_vma_lock_write(vma);
                i_mmap_lock_write(vma->vm_file->f_mapping);
        } else {
                i_mmap_assert_write_locked(vma->vm_file->f_mapping);
        }
        for (address = start; address < end; address += PUD_SIZE) {
                ptep = hugetlb_walk(vma, address, sz);
                if (!ptep)
                        continue;
                ptl = huge_pte_lock(h, mm, ptep);
                __huge_pmd_unshare(&tlb, vma, address, ptep, take_locks);
                spin_unlock(ptl);
        }
        huge_pmd_unshare_flush(&tlb, vma);
        if (take_locks) {
                i_mmap_unlock_write(vma->vm_file->f_mapping);
                hugetlb_vma_unlock_write(vma);
        }
        /*
         * No need to call mmu_notifier_arch_invalidate_secondary_tlbs(), see
         * Documentation/mm/mmu_notifier.rst.
         */
        mmu_notifier_invalidate_range_end(&range);
        tlb_finish_mmu(&tlb);
}

/*
 * This function will unconditionally remove all the shared pmd pgtable entries
 * within the specific vma for a hugetlbfs memory range.
 */
void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
{
        hugetlb_unshare_pmds(vma, ALIGN(vma->vm_start, PUD_SIZE),
                        ALIGN_DOWN(vma->vm_end, PUD_SIZE),
                        /* take_locks = */ true);
}

/*
 * For hugetlb, mremap() is an odd edge case - while the VMA copying is
 * performed, we permit both the old and new VMAs to reference the same
 * reservation.
 *
 * We fix this up after the operation succeeds, or if a newly allocated VMA
 * is closed as a result of a failure to allocate memory.
 */
void fixup_hugetlb_reservations(struct vm_area_struct *vma)
{
        if (is_vm_hugetlb_page(vma))
                clear_vma_resv_huge_pages(vma);
}
































   28 
   28 
   28 




    1 




   27 


    2 
   26 





   27 

   10 
   24 


























































   30 





   30 







   30 







   28 








   10 

   10 







    3 


    1 







    2 




   18 



   18 







    2 



   18 


   18 






   18 











   18 
   12 


   18 











    4 

    1 













    8 



    8 




















    8 






    9 





































    8 
    8 





























































    7 






    7 



    1 
























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 */

#include "queueing.h"
#include "socket.h"
#include "timers.h"
#include "device.h"
#include "ratelimiter.h"
#include "peer.h"
#include "messages.h"

#include <linux/module.h>
#include <linux/rtnetlink.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/inetdevice.h>
#include <linux/if_arp.h>
#include <linux/icmp.h>
#include <linux/suspend.h>
#include <net/dst_metadata.h>
#include <net/gso.h>
#include <net/icmp.h>
#include <net/rtnetlink.h>
#include <net/ip_tunnels.h>
#include <net/addrconf.h>

static LIST_HEAD(device_list);

static int wg_open(struct net_device *dev)
{
        struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
        struct inet6_dev *dev_v6 = __in6_dev_get(dev);
        struct wg_device *wg = netdev_priv(dev);
        struct wg_peer *peer;
        int ret;

        if (dev_v4) {
                /* At some point we might put this check near the ip_rt_send_
                 * redirect call of ip_forward in net/ipv4/ip_forward.c, similar
                 * to the current secpath check.
                 */
                IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
                IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
        }
        if (dev_v6)
                dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;

        mutex_lock(&wg->device_update_lock);
        ret = wg_socket_init(wg, wg->incoming_port);
        if (ret < 0)
                goto out;
        list_for_each_entry(peer, &wg->peer_list, peer_list) {
                wg_packet_send_staged_packets(peer);
                if (peer->persistent_keepalive_interval)
                        wg_packet_send_keepalive(peer);
        }
out:
        mutex_unlock(&wg->device_update_lock);
        return ret;
}

static int wg_pm_notification(struct notifier_block *nb, unsigned long action, void *data)
{
        struct wg_device *wg;
        struct wg_peer *peer;

        /* If the machine is constantly suspending and resuming, as part of
         * its normal operation rather than as a somewhat rare event, then we
         * don't actually want to clear keys.
         */
        if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) ||
            IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP))
                return 0;

        if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
                return 0;

        rtnl_lock();
        list_for_each_entry(wg, &device_list, device_list) {
                mutex_lock(&wg->device_update_lock);
                list_for_each_entry(peer, &wg->peer_list, peer_list) {
                        timer_delete(&peer->timer_zero_key_material);
                        wg_noise_handshake_clear(&peer->handshake);
                        wg_noise_keypairs_clear(&peer->keypairs);
                }
                mutex_unlock(&wg->device_update_lock);
        }
        rtnl_unlock();
        rcu_barrier();
        return 0;
}

static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };

static int wg_vm_notification(struct notifier_block *nb, unsigned long action, void *data)
{
        struct wg_device *wg;
        struct wg_peer *peer;

        rtnl_lock();
        list_for_each_entry(wg, &device_list, device_list) {
                mutex_lock(&wg->device_update_lock);
                list_for_each_entry(peer, &wg->peer_list, peer_list)
                        wg_noise_expire_current_peer_keypairs(peer);
                mutex_unlock(&wg->device_update_lock);
        }
        rtnl_unlock();
        return 0;
}

static struct notifier_block vm_notifier = { .notifier_call = wg_vm_notification };

static int wg_stop(struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);
        struct wg_peer *peer;
        struct sk_buff *skb;

        mutex_lock(&wg->device_update_lock);
        list_for_each_entry(peer, &wg->peer_list, peer_list) {
                wg_packet_purge_staged_packets(peer);
                wg_timers_stop(peer);
                wg_noise_handshake_clear(&peer->handshake);
                wg_noise_keypairs_clear(&peer->keypairs);
                wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
        }
        mutex_unlock(&wg->device_update_lock);
        while ((skb = ptr_ring_consume(&wg->handshake_queue.ring)) != NULL)
                kfree_skb(skb);
        atomic_set(&wg->handshake_queue_len, 0);
        wg_socket_reinit(wg, NULL, NULL);
        return 0;
}

static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);
        struct sk_buff_head packets;
        struct wg_peer *peer;
        struct sk_buff *next;
        sa_family_t family;
        u32 mtu;
        int ret;

        if (unlikely(!wg_check_packet_protocol(skb))) {
                ret = -EPROTONOSUPPORT;
                net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
                goto err;
        }

        peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
        if (unlikely(!peer)) {
                ret = -ENOKEY;
                if (skb->protocol == htons(ETH_P_IP))
                        net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
                                            dev->name, &ip_hdr(skb)->daddr);
                else if (skb->protocol == htons(ETH_P_IPV6))
                        net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
                                            dev->name, &ipv6_hdr(skb)->daddr);
                goto err_icmp;
        }

        family = READ_ONCE(peer->endpoint.addr.sa_family);
        if (unlikely(family != AF_INET && family != AF_INET6)) {
                ret = -EDESTADDRREQ;
                net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
                                    dev->name, peer->internal_id);
                goto err_peer;
        }

        mtu = skb_valid_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;

        __skb_queue_head_init(&packets);
        if (!skb_is_gso(skb)) {
                skb_mark_not_on_list(skb);
        } else {
                struct sk_buff *segs = skb_gso_segment(skb, 0);

                if (IS_ERR(segs)) {
                        ret = PTR_ERR(segs);
                        goto err_peer;
                }
                dev_kfree_skb(skb);
                skb = segs;
        }

        skb_list_walk_safe(skb, skb, next) {
                skb_mark_not_on_list(skb);

                skb = skb_share_check(skb, GFP_ATOMIC);
                if (unlikely(!skb))
                        continue;

                /* We only need to keep the original dst around for icmp,
                 * so at this point we're in a position to drop it.
                 */
                skb_dst_drop(skb);

                PACKET_CB(skb)->mtu = mtu;

                __skb_queue_tail(&packets, skb);
        }

        spin_lock_bh(&peer->staged_packet_queue.lock);
        /* If the queue is getting too big, we start removing the oldest packets
         * until it's small again. We do this before adding the new packet, so
         * we don't remove GSO segments that are in excess.
         */
        while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
                dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
                DEV_STATS_INC(dev, tx_dropped);
        }
        skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
        spin_unlock_bh(&peer->staged_packet_queue.lock);

        wg_packet_send_staged_packets(peer);

        wg_peer_put(peer);
        return NETDEV_TX_OK;

err_peer:
        wg_peer_put(peer);
err_icmp:
        if (skb->protocol == htons(ETH_P_IP))
                icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
        else if (skb->protocol == htons(ETH_P_IPV6))
                icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
err:
        DEV_STATS_INC(dev, tx_errors);
        kfree_skb(skb);
        return ret;
}

static const struct net_device_ops netdev_ops = {
        .ndo_open                = wg_open,
        .ndo_stop                = wg_stop,
        .ndo_start_xmit                = wg_xmit,
};

static void wg_destruct(struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);

        rtnl_lock();
        list_del(&wg->device_list);
        rtnl_unlock();
        mutex_lock(&wg->device_update_lock);
        rcu_assign_pointer(wg->creating_net, NULL);
        wg->incoming_port = 0;
        wg_socket_reinit(wg, NULL, NULL);
        /* The final references are cleared in the below calls to destroy_workqueue. */
        wg_peer_remove_all(wg);
        destroy_workqueue(wg->handshake_receive_wq);
        destroy_workqueue(wg->handshake_send_wq);
        destroy_workqueue(wg->packet_crypt_wq);
        wg_packet_queue_free(&wg->handshake_queue, true);
        wg_packet_queue_free(&wg->decrypt_queue, false);
        wg_packet_queue_free(&wg->encrypt_queue, false);
        rcu_barrier(); /* Wait for all the peers to be actually freed. */
        wg_ratelimiter_uninit();
        memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
        kvfree(wg->index_hashtable);
        kvfree(wg->peer_hashtable);
        mutex_unlock(&wg->device_update_lock);

        pr_debug("%s: Interface destroyed\n", dev->name);
        free_netdev(dev);
}

static const struct device_type device_type = { .name = KBUILD_MODNAME };

static void wg_setup(struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);
        enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
                                    NETIF_F_SG | NETIF_F_GSO |
                                    NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA };
        const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) +
                             max(sizeof(struct ipv6hdr), sizeof(struct iphdr));

        dev->netdev_ops = &netdev_ops;
        dev->header_ops = &ip_tunnel_header_ops;
        dev->hard_header_len = 0;
        dev->addr_len = 0;
        dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
        dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
        dev->type = ARPHRD_NONE;
        dev->flags = IFF_POINTOPOINT | IFF_NOARP;
        dev->priv_flags |= IFF_NO_QUEUE;
        dev->lltx = true;
        dev->features |= WG_NETDEV_FEATURES;
        dev->hw_features |= WG_NETDEV_FEATURES;
        dev->hw_enc_features |= WG_NETDEV_FEATURES;
        dev->mtu = ETH_DATA_LEN - overhead;
        dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead;
        dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;

        SET_NETDEV_DEVTYPE(dev, &device_type);

        /* We need to keep the dst around in case of icmp replies. */
        netif_keep_dst(dev);

        netif_set_tso_max_size(dev, GSO_MAX_SIZE);

        wg->dev = dev;
}

static int wg_newlink(struct net_device *dev,
                      struct rtnl_newlink_params *params,
                      struct netlink_ext_ack *extack)
{
        struct net *link_net = rtnl_newlink_link_net(params);
        struct wg_device *wg = netdev_priv(dev);
        int ret = -ENOMEM;

        rcu_assign_pointer(wg->creating_net, link_net);
        init_rwsem(&wg->static_identity.lock);
        mutex_init(&wg->socket_update_lock);
        mutex_init(&wg->device_update_lock);
        wg_allowedips_init(&wg->peer_allowedips);
        wg_cookie_checker_init(&wg->cookie_checker, wg);
        INIT_LIST_HEAD(&wg->peer_list);
        wg->device_update_gen = 1;

        wg->peer_hashtable = wg_pubkey_hashtable_alloc();
        if (!wg->peer_hashtable)
                return ret;

        wg->index_hashtable = wg_index_hashtable_alloc();
        if (!wg->index_hashtable)
                goto err_free_peer_hashtable;

        wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
                        WQ_CPU_INTENSIVE | WQ_FREEZABLE | WQ_PERCPU, 0,
                        dev->name);
        if (!wg->handshake_receive_wq)
                goto err_free_index_hashtable;

        wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
                        WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
        if (!wg->handshake_send_wq)
                goto err_destroy_handshake_receive;

        wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
                        WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_PERCPU, 0,
                        dev->name);
        if (!wg->packet_crypt_wq)
                goto err_destroy_handshake_send;

        ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
                                   MAX_QUEUED_PACKETS);
        if (ret < 0)
                goto err_destroy_packet_crypt;

        ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
                                   MAX_QUEUED_PACKETS);
        if (ret < 0)
                goto err_free_encrypt_queue;

        ret = wg_packet_queue_init(&wg->handshake_queue, wg_packet_handshake_receive_worker,
                                   MAX_QUEUED_INCOMING_HANDSHAKES);
        if (ret < 0)
                goto err_free_decrypt_queue;

        ret = wg_ratelimiter_init();
        if (ret < 0)
                goto err_free_handshake_queue;

        netif_threaded_enable(dev);
        ret = register_netdevice(dev);
        if (ret < 0)
                goto err_uninit_ratelimiter;

        list_add(&wg->device_list, &device_list);

        /* We wait until the end to assign priv_destructor, so that
         * register_netdevice doesn't call it for us if it fails.
         */
        dev->priv_destructor = wg_destruct;

        pr_debug("%s: Interface created\n", dev->name);
        return ret;

err_uninit_ratelimiter:
        wg_ratelimiter_uninit();
err_free_handshake_queue:
        wg_packet_queue_free(&wg->handshake_queue, false);
err_free_decrypt_queue:
        wg_packet_queue_free(&wg->decrypt_queue, false);
err_free_encrypt_queue:
        wg_packet_queue_free(&wg->encrypt_queue, false);
err_destroy_packet_crypt:
        destroy_workqueue(wg->packet_crypt_wq);
err_destroy_handshake_send:
        destroy_workqueue(wg->handshake_send_wq);
err_destroy_handshake_receive:
        destroy_workqueue(wg->handshake_receive_wq);
err_free_index_hashtable:
        kvfree(wg->index_hashtable);
err_free_peer_hashtable:
        kvfree(wg->peer_hashtable);
        return ret;
}

static struct rtnl_link_ops link_ops __read_mostly = {
        .kind                        = KBUILD_MODNAME,
        .priv_size                = sizeof(struct wg_device),
        .setup                        = wg_setup,
        .newlink                = wg_newlink,
};

static void __net_exit wg_netns_exit_rtnl(struct net *net, struct list_head *dev_kill_list)
{
        struct wg_device *wg;
        struct wg_peer *peer;

        list_for_each_entry(wg, &device_list, device_list) {
                if (rcu_access_pointer(wg->creating_net) == net) {
                        pr_debug("%s: Creating namespace exiting\n", wg->dev->name);
                        netif_carrier_off(wg->dev);
                        mutex_lock(&wg->device_update_lock);
                        rcu_assign_pointer(wg->creating_net, NULL);
                        wg_socket_reinit(wg, NULL, NULL);
                        list_for_each_entry(peer, &wg->peer_list, peer_list)
                                wg_socket_clear_peer_endpoint_src(peer);
                        mutex_unlock(&wg->device_update_lock);
                }
        }
}

static struct pernet_operations pernet_ops __read_mostly = {
        .exit_rtnl = wg_netns_exit_rtnl
};

int __init wg_device_init(void)
{
        int ret;

        ret = register_pm_notifier(&pm_notifier);
        if (ret)
                return ret;

        ret = register_random_vmfork_notifier(&vm_notifier);
        if (ret)
                goto error_pm;

        ret = register_pernet_device(&pernet_ops);
        if (ret)
                goto error_vm;

        ret = rtnl_link_register(&link_ops);
        if (ret)
                goto error_pernet;

        return 0;

error_pernet:
        unregister_pernet_device(&pernet_ops);
error_vm:
        unregister_random_vmfork_notifier(&vm_notifier);
error_pm:
        unregister_pm_notifier(&pm_notifier);
        return ret;
}

void wg_device_uninit(void)
{
        rtnl_link_unregister(&link_ops);
        unregister_pernet_device(&pernet_ops);
        unregister_random_vmfork_notifier(&vm_notifier);
        unregister_pm_notifier(&pm_notifier);
        rcu_barrier();
}










































































    2 




















    4 
    1 















    4 



    4 






    4 
    4 










    4 
    3 
    2 















    1 
    1 
    1 




    4 










    3 





























    3 
































    1 











    3 






    1 













    1 



    1 









    3 


    3 







    2 
    3 





    4 


    2 
    2 












































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
// SPDX-License-Identifier: GPL-2.0-only
/*
 * TCP Vegas congestion control
 *
 * This is based on the congestion detection/avoidance scheme described in
 *    Lawrence S. Brakmo and Larry L. Peterson.
 *    "TCP Vegas: End to end congestion avoidance on a global internet."
 *    IEEE Journal on Selected Areas in Communication, 13(8):1465--1480,
 *    October 1995. Available from:
 *        ftp://ftp.cs.arizona.edu/xkernel/Papers/jsac.ps
 *
 * See http://www.cs.arizona.edu/xkernel/ for their implementation.
 * The main aspects that distinguish this implementation from the
 * Arizona Vegas implementation are:
 *   o We do not change the loss detection or recovery mechanisms of
 *     Linux in any way. Linux already recovers from losses quite well,
 *     using fine-grained timers, NewReno, and FACK.
 *   o To avoid the performance penalty imposed by increasing cwnd
 *     only every-other RTT during slow start, we increase during
 *     every RTT during slow start, just like Reno.
 *   o Largely to allow continuous cwnd growth during slow start,
 *     we use the rate at which ACKs come back as the "actual"
 *     rate, rather than the rate at which data is sent.
 *   o To speed convergence to the right rate, we set the cwnd
 *     to achieve the right ("actual") rate when we exit slow start.
 *   o To filter out the noise caused by delayed ACKs, we use the
 *     minimum RTT sample observed during the last RTT to calculate
 *     the actual rate.
 *   o When the sender re-starts from idle, it waits until it has
 *     received ACKs for an entire flight of new data before making
 *     a cwnd adjustment decision. The original Vegas implementation
 *     assumed senders never went idle.
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/inet_diag.h>

#include <net/tcp.h>

#include "tcp_vegas.h"

static int alpha = 2;
static int beta  = 4;
static int gamma = 1;

module_param(alpha, int, 0644);
MODULE_PARM_DESC(alpha, "lower bound of packets in network");
module_param(beta, int, 0644);
MODULE_PARM_DESC(beta, "upper bound of packets in network");
module_param(gamma, int, 0644);
MODULE_PARM_DESC(gamma, "limit on increase (scale by 2)");

/* There are several situations when we must "re-start" Vegas:
 *
 *  o when a connection is established
 *  o after an RTO
 *  o after fast recovery
 *  o when we send a packet and there is no outstanding
 *    unacknowledged data (restarting an idle connection)
 *
 * In these circumstances we cannot do a Vegas calculation at the
 * end of the first RTT, because any calculation we do is using
 * stale info -- both the saved cwnd and congestion feedback are
 * stale.
 *
 * Instead we must wait until the completion of an RTT during
 * which we actually receive ACKs.
 */
static void vegas_enable(struct sock *sk)
{
        const struct tcp_sock *tp = tcp_sk(sk);
        struct vegas *vegas = inet_csk_ca(sk);

        /* Begin taking Vegas samples next time we send something. */
        vegas->doing_vegas_now = 1;

        /* Set the beginning of the next send window. */
        vegas->beg_snd_nxt = tp->snd_nxt;

        vegas->cntRTT = 0;
        vegas->minRTT = 0x7fffffff;
}

/* Stop taking Vegas samples for now. */
static inline void vegas_disable(struct sock *sk)
{
        struct vegas *vegas = inet_csk_ca(sk);

        vegas->doing_vegas_now = 0;
}

void tcp_vegas_init(struct sock *sk)
{
        struct vegas *vegas = inet_csk_ca(sk);

        vegas->baseRTT = 0x7fffffff;
        vegas_enable(sk);
}
EXPORT_SYMBOL_GPL(tcp_vegas_init);

/* Do RTT sampling needed for Vegas.
 * Basically we:
 *   o min-filter RTT samples from within an RTT to get the current
 *     propagation delay + queuing delay (we are min-filtering to try to
 *     avoid the effects of delayed ACKs)
 *   o min-filter RTT samples from a much longer window (forever for now)
 *     to find the propagation delay (baseRTT)
 */
void tcp_vegas_pkts_acked(struct sock *sk, const struct ack_sample *sample)
{
        struct vegas *vegas = inet_csk_ca(sk);
        u32 vrtt;

        if (sample->rtt_us < 0)
                return;

        /* Never allow zero rtt or baseRTT */
        vrtt = sample->rtt_us + 1;

        /* Filter to find propagation delay: */
        if (vrtt < vegas->baseRTT)
                vegas->baseRTT = vrtt;

        /* Find the min RTT during the last RTT to find
         * the current prop. delay + queuing delay:
         */
        vegas->minRTT = min(vegas->minRTT, vrtt);
        vegas->cntRTT++;
}
EXPORT_SYMBOL_GPL(tcp_vegas_pkts_acked);

void tcp_vegas_state(struct sock *sk, u8 ca_state)
{
        if (ca_state == TCP_CA_Open)
                vegas_enable(sk);
        else
                vegas_disable(sk);
}
EXPORT_SYMBOL_GPL(tcp_vegas_state);

/*
 * If the connection is idle and we are restarting,
 * then we don't want to do any Vegas calculations
 * until we get fresh RTT samples.  So when we
 * restart, we reset our Vegas state to a clean
 * slate. After we get acks for this flight of
 * packets, _then_ we can make Vegas calculations
 * again.
 */
void tcp_vegas_cwnd_event(struct sock *sk, enum tcp_ca_event event)
{
        if (event == CA_EVENT_CWND_RESTART)
                tcp_vegas_init(sk);
}
EXPORT_SYMBOL_GPL(tcp_vegas_cwnd_event);

void tcp_vegas_cwnd_event_tx_start(struct sock *sk)
{
        tcp_vegas_init(sk);
}
EXPORT_SYMBOL_GPL(tcp_vegas_cwnd_event_tx_start);

static inline u32 tcp_vegas_ssthresh(struct tcp_sock *tp)
{
        return  min(tp->snd_ssthresh, tcp_snd_cwnd(tp));
}

static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 acked)
{
        struct tcp_sock *tp = tcp_sk(sk);
        struct vegas *vegas = inet_csk_ca(sk);

        if (!vegas->doing_vegas_now) {
                tcp_reno_cong_avoid(sk, ack, acked);
                return;
        }

        if (after(ack, vegas->beg_snd_nxt)) {
                /* Do the Vegas once-per-RTT cwnd adjustment. */

                /* Save the extent of the current window so we can use this
                 * at the end of the next RTT.
                 */
                vegas->beg_snd_nxt  = tp->snd_nxt;

                /* We do the Vegas calculations only if we got enough RTT
                 * samples that we can be reasonably sure that we got
                 * at least one RTT sample that wasn't from a delayed ACK.
                 * If we only had 2 samples total,
                 * then that means we're getting only 1 ACK per RTT, which
                 * means they're almost certainly delayed ACKs.
                 * If  we have 3 samples, we should be OK.
                 */

                if (vegas->cntRTT <= 2) {
                        /* We don't have enough RTT samples to do the Vegas
                         * calculation, so we'll behave like Reno.
                         */
                        tcp_reno_cong_avoid(sk, ack, acked);
                } else {
                        u32 rtt, diff;
                        u64 target_cwnd;

                        /* We have enough RTT samples, so, using the Vegas
                         * algorithm, we determine if we should increase or
                         * decrease cwnd, and by how much.
                         */

                        /* Pluck out the RTT we are using for the Vegas
                         * calculations. This is the min RTT seen during the
                         * last RTT. Taking the min filters out the effects
                         * of delayed ACKs, at the cost of noticing congestion
                         * a bit later.
                         */
                        rtt = vegas->minRTT;

                        /* Calculate the cwnd we should have, if we weren't
                         * going too fast.
                         *
                         * This is:
                         *     (actual rate in segments) * baseRTT
                         */
                        target_cwnd = (u64)tcp_snd_cwnd(tp) * vegas->baseRTT;
                        do_div(target_cwnd, rtt);

                        /* Calculate the difference between the window we had,
                         * and the window we would like to have. This quantity
                         * is the "Diff" from the Arizona Vegas papers.
                         */
                        diff = tcp_snd_cwnd(tp) * (rtt-vegas->baseRTT) / vegas->baseRTT;

                        if (diff > gamma && tcp_in_slow_start(tp)) {
                                /* Going too fast. Time to slow down
                                 * and switch to congestion avoidance.
                                 */

                                /* Set cwnd to match the actual rate
                                 * exactly:
                                 *   cwnd = (actual rate) * baseRTT
                                 * Then we add 1 because the integer
                                 * truncation robs us of full link
                                 * utilization.
                                 */
                                tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp),
                                                         (u32)target_cwnd + 1));
                                WRITE_ONCE(tp->snd_ssthresh,
                                           tcp_vegas_ssthresh(tp));

                        } else if (tcp_in_slow_start(tp)) {
                                /* Slow start.  */
                                tcp_slow_start(tp, acked);
                        } else {
                                /* Congestion avoidance. */

                                /* Figure out where we would like cwnd
                                 * to be.
                                 */
                                if (diff > beta) {
                                        /* The old window was too fast, so
                                         * we slow down.
                                         */
                                        tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - 1);
                                        WRITE_ONCE(tp->snd_ssthresh,
                                                   tcp_vegas_ssthresh(tp));
                                } else if (diff < alpha) {
                                        /* We don't have enough extra packets
                                         * in the network, so speed up.
                                         */
                                        tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1);
                                } else {
                                        /* Sending just as fast as we
                                         * should be.
                                         */
                                }
                        }

                        if (tcp_snd_cwnd(tp) < 2)
                                tcp_snd_cwnd_set(tp, 2);
                        else if (tcp_snd_cwnd(tp) > tp->snd_cwnd_clamp)
                                tcp_snd_cwnd_set(tp, tp->snd_cwnd_clamp);

                        WRITE_ONCE(tp->snd_ssthresh, tcp_current_ssthresh(sk));
                }

                /* Wipe the slate clean for the next RTT. */
                vegas->cntRTT = 0;
                vegas->minRTT = 0x7fffffff;
        }
        /* Use normal slow start */
        else if (tcp_in_slow_start(tp))
                tcp_slow_start(tp, acked);
}

/* Extract info for Tcp socket info provided via netlink. */
size_t tcp_vegas_get_info(struct sock *sk, u32 ext, int *attr,
                          union tcp_cc_info *info)
{
        const struct vegas *ca = inet_csk_ca(sk);

        if (ext & (1 << (INET_DIAG_VEGASINFO - 1))) {
                info->vegas.tcpv_enabled = ca->doing_vegas_now;
                info->vegas.tcpv_rttcnt = ca->cntRTT;
                info->vegas.tcpv_rtt = ca->baseRTT;
                info->vegas.tcpv_minrtt = ca->minRTT;

                *attr = INET_DIAG_VEGASINFO;
                return sizeof(struct tcpvegas_info);
        }
        return 0;
}
EXPORT_SYMBOL_GPL(tcp_vegas_get_info);

static struct tcp_congestion_ops tcp_vegas __read_mostly = {
        .init                = tcp_vegas_init,
        .ssthresh        = tcp_reno_ssthresh,
        .undo_cwnd        = tcp_reno_undo_cwnd,
        .cong_avoid        = tcp_vegas_cong_avoid,
        .pkts_acked        = tcp_vegas_pkts_acked,
        .set_state        = tcp_vegas_state,
        .cwnd_event        = tcp_vegas_cwnd_event,
        .cwnd_event_tx_start = tcp_vegas_cwnd_event_tx_start,
        .get_info        = tcp_vegas_get_info,

        .owner                = THIS_MODULE,
        .name                = "vegas",
};

static int __init tcp_vegas_register(void)
{
        BUILD_BUG_ON(sizeof(struct vegas) > ICSK_CA_PRIV_SIZE);
        tcp_register_congestion_control(&tcp_vegas);
        return 0;
}

static void __exit tcp_vegas_unregister(void)
{
        tcp_unregister_congestion_control(&tcp_vegas);
}

module_init(tcp_vegas_register);
module_exit(tcp_vegas_unregister);

MODULE_AUTHOR("Stephen Hemminger");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("TCP Vegas");




















   19 




    1 



    1 


   16 













   10 









    1 


    5 






    1 



    1 




    2 








    1 




    1 
    1 



    2 




    2 








    5 


    5 


    5 

    1 




    1 

    2 

    1 




    5 
    5 




    5 
    5 


    5 







    5 
























   19 


   19 



   16 



    5 



    5 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2010: YOSHIFUJI Hideaki <yoshfuji@linux-ipv6.org>
 * Copyright (C) 2015: Linus Lüssing <linus.luessing@c0d3.blue>
 *
 * Based on the MLD support added to br_multicast.c by YOSHIFUJI Hideaki.
 */

#include <linux/skbuff.h>
#include <net/ipv6.h>
#include <net/mld.h>
#include <net/addrconf.h>
#include <net/ip6_checksum.h>

static int ipv6_mc_check_ip6hdr(struct sk_buff *skb)
{
        const struct ipv6hdr *ip6h;
        unsigned int len;
        unsigned int offset = skb_network_offset(skb) + sizeof(*ip6h);

        if (!pskb_may_pull(skb, offset))
                return -EINVAL;

        ip6h = ipv6_hdr(skb);

        if (ip6h->version != 6)
                return -EINVAL;

        len = offset + ntohs(ip6h->payload_len);
        if (skb->len < len || len <= offset)
                return -EINVAL;

        skb_set_transport_header(skb, offset);

        return 0;
}

static int ipv6_mc_check_exthdrs(struct sk_buff *skb)
{
        const struct ipv6hdr *ip6h;
        int offset;
        u8 nexthdr;
        __be16 frag_off;

        ip6h = ipv6_hdr(skb);

        if (ip6h->nexthdr != IPPROTO_HOPOPTS)
                return -ENOMSG;

        nexthdr = ip6h->nexthdr;
        offset = skb_network_offset(skb) + sizeof(*ip6h);
        offset = ipv6_skip_exthdr(skb, offset, &nexthdr, &frag_off);

        if (offset < 0)
                return -EINVAL;

        if (nexthdr != IPPROTO_ICMPV6)
                return -ENOMSG;

        skb_set_transport_header(skb, offset);

        return 0;
}

static int ipv6_mc_check_mld_reportv2(struct sk_buff *skb)
{
        unsigned int len = skb_transport_offset(skb);

        len += sizeof(struct mld2_report);

        return ipv6_mc_may_pull(skb, len) ? 0 : -EINVAL;
}

static int ipv6_mc_check_mld_query(struct sk_buff *skb)
{
        unsigned int transport_len = ipv6_transport_len(skb);
        struct mld_msg *mld;
        unsigned int len;

        /* RFC2710+RFC3810 (MLDv1+MLDv2) require link-local source addresses */
        if (!(ipv6_addr_type(&ipv6_hdr(skb)->saddr) & IPV6_ADDR_LINKLOCAL))
                return -EINVAL;

        /* MLDv1? */
        if (transport_len != sizeof(struct mld_msg)) {
                /* or MLDv2? */
                if (transport_len < sizeof(struct mld2_query))
                        return -EINVAL;

                len = skb_transport_offset(skb) + sizeof(struct mld2_query);
                if (!ipv6_mc_may_pull(skb, len))
                        return -EINVAL;
        }

        mld = (struct mld_msg *)skb_transport_header(skb);

        /* RFC2710+RFC3810 (MLDv1+MLDv2) require the multicast link layer
         * all-nodes destination address (ff02::1) for general queries
         */
        if (ipv6_addr_any(&mld->mld_mca) &&
            !ipv6_addr_is_ll_all_nodes(&ipv6_hdr(skb)->daddr))
                return -EINVAL;

        return 0;
}

static int ipv6_mc_check_mld_msg(struct sk_buff *skb)
{
        unsigned int len = skb_transport_offset(skb) + sizeof(struct mld_msg);
        struct mld_msg *mld;

        if (!ipv6_mc_may_pull(skb, len))
                return -ENODATA;

        mld = (struct mld_msg *)skb_transport_header(skb);

        switch (mld->mld_type) {
        case ICMPV6_MGM_REDUCTION:
        case ICMPV6_MGM_REPORT:
                return 0;
        case ICMPV6_MLD2_REPORT:
                return ipv6_mc_check_mld_reportv2(skb);
        case ICMPV6_MGM_QUERY:
                return ipv6_mc_check_mld_query(skb);
        default:
                return -ENODATA;
        }
}

static inline __sum16 ipv6_mc_validate_checksum(struct sk_buff *skb)
{
        return skb_checksum_validate(skb, IPPROTO_ICMPV6, ip6_compute_pseudo);
}

static int ipv6_mc_check_icmpv6(struct sk_buff *skb)
{
        unsigned int len = skb_transport_offset(skb) + sizeof(struct icmp6hdr);
        unsigned int transport_len = ipv6_transport_len(skb);
        struct sk_buff *skb_chk;

        if (!ipv6_mc_may_pull(skb, len))
                return -EINVAL;

        skb_chk = skb_checksum_trimmed(skb, transport_len,
                                       ipv6_mc_validate_checksum);
        if (!skb_chk)
                return -EINVAL;

        if (skb_chk != skb)
                kfree_skb(skb_chk);

        return 0;
}

/**
 * ipv6_mc_check_mld - checks whether this is a sane MLD packet
 * @skb: the skb to validate
 *
 * Checks whether an IPv6 packet is a valid MLD packet. If so sets
 * skb transport header accordingly and returns zero.
 *
 * -EINVAL: A broken packet was detected, i.e. it violates some internet
 *  standard
 * -ENOMSG: IP header validation succeeded but it is not an ICMPv6 packet
 *  with a hop-by-hop option.
 * -ENODATA: IP+ICMPv6 header with hop-by-hop option validation succeeded
 *  but it is not an MLD packet.
 * -ENOMEM: A memory allocation failure happened.
 *
 * Caller needs to set the skb network header and free any returned skb if it
 * differs from the provided skb.
 */
int ipv6_mc_check_mld(struct sk_buff *skb)
{
        int ret;

        ret = ipv6_mc_check_ip6hdr(skb);
        if (ret < 0)
                return ret;

        ret = ipv6_mc_check_exthdrs(skb);
        if (ret < 0)
                return ret;

        ret = ipv6_mc_check_icmpv6(skb);
        if (ret < 0)
                return ret;

        return ipv6_mc_check_mld_msg(skb);
}
EXPORT_SYMBOL(ipv6_mc_check_mld);









































































































































































































































































































































































































































































































 2821 
 2821 










 2821 
 2821 














 1648 









 1660 












































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   23 




   23 







   23 






















   23 







   23 








































































































    9 











    9 















    9 











































































































































































































































































































































































































































































































































































































































































































































































 4280 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
// SPDX-License-Identifier: GPL-2.0
/* CPU control.
 * (C) 2001, 2002, 2003, 2004 Rusty Russell
 */
#include <linux/sched/mm.h>
#include <linux/proc_fs.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/notifier.h>
#include <linux/sched/signal.h>
#include <linux/sched/hotplug.h>
#include <linux/sched/isolation.h>
#include <linux/sched/task.h>
#include <linux/sched/smt.h>
#include <linux/unistd.h>
#include <linux/cpu.h>
#include <linux/oom.h>
#include <linux/rcupdate.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/bug.h>
#include <linux/kthread.h>
#include <linux/stop_machine.h>
#include <linux/mutex.h>
#include <linux/gfp.h>
#include <linux/suspend.h>
#include <linux/lockdep.h>
#include <linux/tick.h>
#include <linux/irq.h>
#include <linux/nmi.h>
#include <linux/smpboot.h>
#include <linux/relay.h>
#include <linux/slab.h>
#include <linux/scs.h>
#include <linux/percpu-rwsem.h>
#include <linux/cpuset.h>
#include <linux/random.h>
#include <linux/cc_platform.h>
#include <linux/parser.h>

#include <trace/events/power.h>
#define CREATE_TRACE_POINTS
#include <trace/events/cpuhp.h>

#include "smpboot.h"

/**
 * struct cpuhp_cpu_state - Per cpu hotplug state storage
 * @state:        The current cpu state
 * @target:        The target state
 * @fail:        Current CPU hotplug callback state
 * @thread:        Pointer to the hotplug thread
 * @should_run:        Thread should execute
 * @rollback:        Perform a rollback
 * @single:        Single callback invocation
 * @bringup:        Single callback bringup or teardown selector
 * @node:        Remote CPU node; for multi-instance, do a
 *                single entry callback for install/remove
 * @last:        For multi-instance rollback, remember how far we got
 * @cb_state:        The state for a single callback (install/uninstall)
 * @result:        Result of the operation
 * @ap_sync_state:        State for AP synchronization
 * @done_up:        Signal completion to the issuer of the task for cpu-up
 * @done_down:        Signal completion to the issuer of the task for cpu-down
 */
struct cpuhp_cpu_state {
        enum cpuhp_state        state;
        enum cpuhp_state        target;
        enum cpuhp_state        fail;
#ifdef CONFIG_SMP
        struct task_struct        *thread;
        bool                        should_run;
        bool                        rollback;
        bool                        single;
        bool                        bringup;
        struct hlist_node        *node;
        struct hlist_node        *last;
        enum cpuhp_state        cb_state;
        int                        result;
        atomic_t                ap_sync_state;
        struct completion        done_up;
        struct completion        done_down;
#endif
};

static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
        .fail = CPUHP_INVALID,
};

#ifdef CONFIG_SMP
cpumask_t cpus_booted_once_mask;
#endif

#if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
static struct lockdep_map cpuhp_state_up_map =
        STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
static struct lockdep_map cpuhp_state_down_map =
        STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);


static inline void cpuhp_lock_acquire(bool bringup)
{
        lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
}

static inline void cpuhp_lock_release(bool bringup)
{
        lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
}
#else

static inline void cpuhp_lock_acquire(bool bringup) { }
static inline void cpuhp_lock_release(bool bringup) { }

#endif

/**
 * struct cpuhp_step - Hotplug state machine step
 * @name:        Name of the step
 * @startup:        Startup function of the step
 * @teardown:        Teardown function of the step
 * @cant_stop:        Bringup/teardown can't be stopped at this step
 * @multi_instance:        State has multiple instances which get added afterwards
 */
struct cpuhp_step {
        const char                *name;
        union {
                int                (*single)(unsigned int cpu);
                int                (*multi)(unsigned int cpu,
                                         struct hlist_node *node);
        } startup;
        union {
                int                (*single)(unsigned int cpu);
                int                (*multi)(unsigned int cpu,
                                         struct hlist_node *node);
        } teardown;
        /* private: */
        struct hlist_head        list;
        /* public: */
        bool                        cant_stop;
        bool                        multi_instance;
};

static DEFINE_MUTEX(cpuhp_state_mutex);
static struct cpuhp_step cpuhp_hp_states[];

static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
{
        return cpuhp_hp_states + state;
}

static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step)
{
        return bringup ? !step->startup.single : !step->teardown.single;
}

/**
 * cpuhp_invoke_callback - Invoke the callbacks for a given state
 * @cpu:        The cpu for which the callback should be invoked
 * @state:        The state to do callbacks for
 * @bringup:        True if the bringup callback should be invoked
 * @node:        For multi-instance, do a single entry callback for install/remove
 * @lastp:        For multi-instance rollback, remember how far we got
 *
 * Called from cpu hotplug and from the state register machinery.
 *
 * Return: %0 on success or a negative errno code
 */
static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
                                 bool bringup, struct hlist_node *node,
                                 struct hlist_node **lastp)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
        struct cpuhp_step *step = cpuhp_get_step(state);
        int (*cbm)(unsigned int cpu, struct hlist_node *node);
        int (*cb)(unsigned int cpu);
        int ret, cnt, rollback_ret;

        if (st->fail == state) {
                st->fail = CPUHP_INVALID;
                return -EAGAIN;
        }

        if (cpuhp_step_empty(bringup, step)) {
                WARN_ON_ONCE(1);
                return 0;
        }

        if (!step->multi_instance) {
                WARN_ON_ONCE(lastp && *lastp);
                cb = bringup ? step->startup.single : step->teardown.single;

                trace_cpuhp_enter(cpu, st->target, state, cb);
                ret = cb(cpu);
                trace_cpuhp_exit(cpu, st->state, state, ret);
                return ret;
        }
        cbm = bringup ? step->startup.multi : step->teardown.multi;

        /* Single invocation for instance add/remove */
        if (node) {
                WARN_ON_ONCE(lastp && *lastp);
                trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
                ret = cbm(cpu, node);
                trace_cpuhp_exit(cpu, st->state, state, ret);
                return ret;
        }

        /* State transition. Invoke on all instances */
        cnt = 0;
        hlist_for_each(node, &step->list) {
                if (lastp && node == *lastp)
                        break;

                trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
                ret = cbm(cpu, node);
                trace_cpuhp_exit(cpu, st->state, state, ret);
                if (ret) {
                        if (!lastp)
                                goto err;

                        *lastp = node;
                        return ret;
                }
                cnt++;
        }
        if (lastp)
                *lastp = NULL;
        return 0;
err:
        /* Rollback the instances if one failed */
        cbm = !bringup ? step->startup.multi : step->teardown.multi;
        if (!cbm)
                return ret;

        hlist_for_each(node, &step->list) {
                if (!cnt--)
                        break;

                trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
                rollback_ret = cbm(cpu, node);
                trace_cpuhp_exit(cpu, st->state, state, rollback_ret);
                /*
                 * Rollback must not fail,
                 */
                WARN_ON_ONCE(rollback_ret);
        }
        return ret;
}

/*
 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
 */
static bool cpuhp_is_atomic_state(enum cpuhp_state state)
{
        return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
}

#ifdef CONFIG_SMP
static bool cpuhp_is_ap_state(enum cpuhp_state state)
{
        /*
         * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
         * purposes as that state is handled explicitly in cpu_down.
         */
        return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
}

static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
{
        struct completion *done = bringup ? &st->done_up : &st->done_down;
        wait_for_completion(done);
}

static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
{
        struct completion *done = bringup ? &st->done_up : &st->done_down;
        complete(done);
}

/* Synchronization state management */
enum cpuhp_sync_state {
        SYNC_STATE_DEAD,
        SYNC_STATE_KICKED,
        SYNC_STATE_SHOULD_DIE,
        SYNC_STATE_ALIVE,
        SYNC_STATE_SHOULD_ONLINE,
        SYNC_STATE_ONLINE,
};

#ifdef CONFIG_HOTPLUG_CORE_SYNC
/**
 * cpuhp_ap_update_sync_state - Update synchronization state during bringup/teardown
 * @state:        The synchronization state to set
 *
 * No synchronization point. Just update of the synchronization state, but implies
 * a full barrier so that the AP changes are visible before the control CPU proceeds.
 */
static inline void cpuhp_ap_update_sync_state(enum cpuhp_sync_state state)
{
        atomic_t *st = this_cpu_ptr(&cpuhp_state.ap_sync_state);

        (void)atomic_xchg(st, state);
}

void __weak arch_cpuhp_sync_state_poll(void) { cpu_relax(); }

static bool cpuhp_wait_for_sync_state(unsigned int cpu, enum cpuhp_sync_state state,
                                      enum cpuhp_sync_state next_state)
{
        atomic_t *st = per_cpu_ptr(&cpuhp_state.ap_sync_state, cpu);
        ktime_t now, end, start = ktime_get();
        int sync;

        end = start + 10ULL * NSEC_PER_SEC;

        sync = atomic_read(st);
        while (1) {
                if (sync == state) {
                        if (!atomic_try_cmpxchg(st, &sync, next_state))
                                continue;
                        return true;
                }

                now = ktime_get();
                if (now > end) {
                        /* Timeout. Leave the state unchanged */
                        return false;
                } else if (now - start < NSEC_PER_MSEC) {
                        /* Poll for one millisecond */
                        arch_cpuhp_sync_state_poll();
                } else {
                        usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC);
                }
                sync = atomic_read(st);
        }
        return true;
}
#else  /* CONFIG_HOTPLUG_CORE_SYNC */
static inline void cpuhp_ap_update_sync_state(enum cpuhp_sync_state state) { }
#endif /* !CONFIG_HOTPLUG_CORE_SYNC */

#ifdef CONFIG_HOTPLUG_CORE_SYNC_DEAD
/**
 * cpuhp_ap_report_dead - Update synchronization state to DEAD
 *
 * No synchronization point. Just update of the synchronization state.
 */
void cpuhp_ap_report_dead(void)
{
        cpuhp_ap_update_sync_state(SYNC_STATE_DEAD);
}

void __weak arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) { }

/*
 * Late CPU shutdown synchronization point. Cannot use cpuhp_state::done_down
 * because the AP cannot issue complete() at this stage.
 */
static void cpuhp_bp_sync_dead(unsigned int cpu)
{
        atomic_t *st = per_cpu_ptr(&cpuhp_state.ap_sync_state, cpu);
        int sync = atomic_read(st);

        do {
                /* CPU can have reported dead already. Don't overwrite that! */
                if (sync == SYNC_STATE_DEAD)
                        break;
        } while (!atomic_try_cmpxchg(st, &sync, SYNC_STATE_SHOULD_DIE));

        if (cpuhp_wait_for_sync_state(cpu, SYNC_STATE_DEAD, SYNC_STATE_DEAD)) {
                /* CPU reached dead state. Invoke the cleanup function */
                arch_cpuhp_cleanup_dead_cpu(cpu);
                return;
        }

        /* No further action possible. Emit message and give up. */
        pr_err("CPU%u failed to report dead state\n", cpu);
}
#else /* CONFIG_HOTPLUG_CORE_SYNC_DEAD */
static inline void cpuhp_bp_sync_dead(unsigned int cpu) { }
#endif /* !CONFIG_HOTPLUG_CORE_SYNC_DEAD */

#ifdef CONFIG_HOTPLUG_CORE_SYNC_FULL
/**
 * cpuhp_ap_sync_alive - Synchronize AP with the control CPU once it is alive
 *
 * Updates the AP synchronization state to SYNC_STATE_ALIVE and waits
 * for the BP to release it.
 */
void cpuhp_ap_sync_alive(void)
{
        atomic_t *st = this_cpu_ptr(&cpuhp_state.ap_sync_state);

        cpuhp_ap_update_sync_state(SYNC_STATE_ALIVE);

        /* Wait for the control CPU to release it. */
        while (atomic_read(st) != SYNC_STATE_SHOULD_ONLINE)
                cpu_relax();
}

static bool cpuhp_can_boot_ap(unsigned int cpu)
{
        atomic_t *st = per_cpu_ptr(&cpuhp_state.ap_sync_state, cpu);
        int sync = atomic_read(st);

again:
        switch (sync) {
        case SYNC_STATE_DEAD:
                /* CPU is properly dead */
                break;
        case SYNC_STATE_KICKED:
                /* CPU did not come up in previous attempt */
                break;
        case SYNC_STATE_ALIVE:
                /* CPU is stuck cpuhp_ap_sync_alive(). */
                break;
        default:
                /* CPU failed to report online or dead and is in limbo state. */
                return false;
        }

        /* Prepare for booting */
        if (!atomic_try_cmpxchg(st, &sync, SYNC_STATE_KICKED))
                goto again;

        return true;
}

void __weak arch_cpuhp_cleanup_kick_cpu(unsigned int cpu) { }

/*
 * Early CPU bringup synchronization point. Cannot use cpuhp_state::done_up
 * because the AP cannot issue complete() so early in the bringup.
 */
static int cpuhp_bp_sync_alive(unsigned int cpu)
{
        int ret = 0;

        if (!IS_ENABLED(CONFIG_HOTPLUG_CORE_SYNC_FULL))
                return 0;

        if (!cpuhp_wait_for_sync_state(cpu, SYNC_STATE_ALIVE, SYNC_STATE_SHOULD_ONLINE)) {
                pr_err("CPU%u failed to report alive state\n", cpu);
                ret = -EIO;
        }

        /* Let the architecture cleanup the kick alive mechanics. */
        arch_cpuhp_cleanup_kick_cpu(cpu);
        return ret;
}
#else /* CONFIG_HOTPLUG_CORE_SYNC_FULL */
static inline int cpuhp_bp_sync_alive(unsigned int cpu) { return 0; }
static inline bool cpuhp_can_boot_ap(unsigned int cpu) { return true; }
#endif /* !CONFIG_HOTPLUG_CORE_SYNC_FULL */

/* Serializes the updates to cpu_online_mask, cpu_present_mask */
static DEFINE_MUTEX(cpu_add_remove_lock);
bool cpuhp_tasks_frozen;
EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);

/*
 * The following two APIs (cpu_maps_update_begin/done) must be used when
 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
 */
void cpu_maps_update_begin(void)
{
        mutex_lock(&cpu_add_remove_lock);
}

void cpu_maps_update_done(void)
{
        mutex_unlock(&cpu_add_remove_lock);
}

/*
 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
 * Should always be manipulated under cpu_add_remove_lock
 */
static int cpu_hotplug_disabled;

#ifdef CONFIG_HOTPLUG_CPU

DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);

static bool cpu_hotplug_offline_disabled __ro_after_init;

void cpus_read_lock(void)
{
        percpu_down_read(&cpu_hotplug_lock);
}
EXPORT_SYMBOL_GPL(cpus_read_lock);

int cpus_read_trylock(void)
{
        return percpu_down_read_trylock(&cpu_hotplug_lock);
}
EXPORT_SYMBOL_GPL(cpus_read_trylock);

void cpus_read_unlock(void)
{
        percpu_up_read(&cpu_hotplug_lock);
}
EXPORT_SYMBOL_GPL(cpus_read_unlock);

void cpus_write_lock(void)
{
        percpu_down_write(&cpu_hotplug_lock);
}

void cpus_write_unlock(void)
{
        percpu_up_write(&cpu_hotplug_lock);
}

void lockdep_assert_cpus_held(void)
{
        /*
         * We can't have hotplug operations before userspace starts running,
         * and some init codepaths will knowingly not take the hotplug lock.
         * This is all valid, so mute lockdep until it makes sense to report
         * unheld locks.
         */
        if (system_state < SYSTEM_RUNNING)
                return;

        percpu_rwsem_assert_held(&cpu_hotplug_lock);
}
EXPORT_SYMBOL_GPL(lockdep_assert_cpus_held);

#ifdef CONFIG_LOCKDEP
int lockdep_is_cpus_held(void)
{
        return percpu_rwsem_is_held(&cpu_hotplug_lock);
}

int lockdep_is_cpus_write_held(void)
{
        return percpu_rwsem_is_write_held(&cpu_hotplug_lock);
}
#endif

static void lockdep_acquire_cpus_lock(void)
{
        rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
}

static void lockdep_release_cpus_lock(void)
{
        rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
}

/* Declare CPU offlining not supported */
void cpu_hotplug_disable_offlining(void)
{
        cpu_maps_update_begin();
        cpu_hotplug_offline_disabled = true;
        cpu_maps_update_done();
}

/*
 * Wait for currently running CPU hotplug operations to complete (if any) and
 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 * hotplug path before performing hotplug operations. So acquiring that lock
 * guarantees mutual exclusion from any currently running hotplug operations.
 */
void cpu_hotplug_disable(void)
{
        cpu_maps_update_begin();
        cpu_hotplug_disabled++;
        cpu_maps_update_done();
}
EXPORT_SYMBOL_GPL(cpu_hotplug_disable);

static void __cpu_hotplug_enable(void)
{
        if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
                return;
        cpu_hotplug_disabled--;
}

void cpu_hotplug_enable(void)
{
        cpu_maps_update_begin();
        __cpu_hotplug_enable();
        cpu_maps_update_done();
}
EXPORT_SYMBOL_GPL(cpu_hotplug_enable);

#else

static void lockdep_acquire_cpus_lock(void)
{
}

static void lockdep_release_cpus_lock(void)
{
}

#endif        /* CONFIG_HOTPLUG_CPU */

/*
 * Architectures that need SMT-specific errata handling during SMT hotplug
 * should override this.
 */
void __weak arch_smt_update(void) { }

#ifdef CONFIG_HOTPLUG_SMT

enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
static unsigned int cpu_smt_max_threads __ro_after_init;
unsigned int cpu_smt_num_threads __read_mostly = UINT_MAX;

void __init cpu_smt_disable(bool force)
{
        if (!cpu_smt_possible())
                return;

        if (force) {
                pr_info("SMT: Force disabled\n");
                cpu_smt_control = CPU_SMT_FORCE_DISABLED;
        } else {
                pr_info("SMT: disabled\n");
                cpu_smt_control = CPU_SMT_DISABLED;
        }
        cpu_smt_num_threads = 1;
}

/*
 * The decision whether SMT is supported can only be done after the full
 * CPU identification. Called from architecture code.
 */
void __init cpu_smt_set_num_threads(unsigned int num_threads,
                                    unsigned int max_threads)
{
        WARN_ON(!num_threads || (num_threads > max_threads));

        if (max_threads == 1)
                cpu_smt_control = CPU_SMT_NOT_SUPPORTED;

        cpu_smt_max_threads = max_threads;

        /*
         * If SMT has been disabled via the kernel command line or SMT is
         * not supported, set cpu_smt_num_threads to 1 for consistency.
         * If enabled, take the architecture requested number of threads
         * to bring up into account.
         */
        if (cpu_smt_control != CPU_SMT_ENABLED)
                cpu_smt_num_threads = 1;
        else if (num_threads < cpu_smt_num_threads)
                cpu_smt_num_threads = num_threads;
}

static int __init smt_cmdline_disable(char *str)
{
        cpu_smt_disable(str && !strcmp(str, "force"));
        return 0;
}
early_param("nosmt", smt_cmdline_disable);

/*
 * For Archicture supporting partial SMT states check if the thread is allowed.
 * Otherwise this has already been checked through cpu_smt_max_threads when
 * setting the SMT level.
 */
static inline bool cpu_smt_thread_allowed(unsigned int cpu)
{
#ifdef CONFIG_SMT_NUM_THREADS_DYNAMIC
        return topology_smt_thread_allowed(cpu);
#else
        return true;
#endif
}

static inline bool cpu_bootable(unsigned int cpu)
{
        if (cpu_smt_control == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu))
                return true;

        /* All CPUs are bootable if controls are not configured */
        if (cpu_smt_control == CPU_SMT_NOT_IMPLEMENTED)
                return true;

        /* All CPUs are bootable if CPU is not SMT capable */
        if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
                return true;

        if (topology_is_primary_thread(cpu))
                return true;

        /*
         * On x86 it's required to boot all logical CPUs at least once so
         * that the init code can get a chance to set CR4.MCE on each
         * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
         * core will shutdown the machine.
         */
        return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
}

/* Returns true if SMT is supported and not forcefully (irreversibly) disabled */
bool cpu_smt_possible(void)
{
        return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
                cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
}
EXPORT_SYMBOL_GPL(cpu_smt_possible);

#else
static inline bool cpu_bootable(unsigned int cpu) { return true; }
#endif

static inline enum cpuhp_state
cpuhp_set_state(int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target)
{
        enum cpuhp_state prev_state = st->state;
        bool bringup = st->state < target;

        st->rollback = false;
        st->last = NULL;

        st->target = target;
        st->single = false;
        st->bringup = bringup;
        if (cpu_dying(cpu) != !bringup)
                set_cpu_dying(cpu, !bringup);

        return prev_state;
}

static inline void
cpuhp_reset_state(int cpu, struct cpuhp_cpu_state *st,
                  enum cpuhp_state prev_state)
{
        bool bringup = !st->bringup;

        st->target = prev_state;

        /*
         * Already rolling back. No need invert the bringup value or to change
         * the current state.
         */
        if (st->rollback)
                return;

        st->rollback = true;

        /*
         * If we have st->last we need to undo partial multi_instance of this
         * state first. Otherwise start undo at the previous state.
         */
        if (!st->last) {
                if (st->bringup)
                        st->state--;
                else
                        st->state++;
        }

        st->bringup = bringup;
        if (cpu_dying(cpu) != !bringup)
                set_cpu_dying(cpu, !bringup);
}

/* Regular hotplug invocation of the AP hotplug thread */
static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
{
        if (!st->single && st->state == st->target)
                return;

        st->result = 0;
        /*
         * Make sure the above stores are visible before should_run becomes
         * true. Paired with the mb() above in cpuhp_thread_fun()
         */
        smp_mb();
        st->should_run = true;
        wake_up_process(st->thread);
        wait_for_ap_thread(st, st->bringup);
}

static int cpuhp_kick_ap(int cpu, struct cpuhp_cpu_state *st,
                         enum cpuhp_state target)
{
        enum cpuhp_state prev_state;
        int ret;

        prev_state = cpuhp_set_state(cpu, st, target);
        __cpuhp_kick_ap(st);
        if ((ret = st->result)) {
                cpuhp_reset_state(cpu, st, prev_state);
                __cpuhp_kick_ap(st);
        }

        return ret;
}

static int bringup_wait_for_ap_online(unsigned int cpu)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);

        /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
        wait_for_ap_thread(st, true);
        if (WARN_ON_ONCE((!cpu_online(cpu))))
                return -ECANCELED;

        /* Unpark the hotplug thread of the target cpu */
        kthread_unpark(st->thread);

        /*
         * SMT soft disabling on X86 requires to bring the CPU out of the
         * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
         * CPU marked itself as booted_once in notify_cpu_starting() so the
         * cpu_bootable() check will now return false if this is not the
         * primary sibling.
         */
        if (!cpu_bootable(cpu))
                return -ECANCELED;
        return 0;
}

#ifdef CONFIG_HOTPLUG_SPLIT_STARTUP
static int cpuhp_kick_ap_alive(unsigned int cpu)
{
        if (!cpuhp_can_boot_ap(cpu))
                return -EAGAIN;

        return arch_cpuhp_kick_ap_alive(cpu, idle_thread_get(cpu));
}

static int cpuhp_bringup_ap(unsigned int cpu)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
        int ret;

        /*
         * Some architectures have to walk the irq descriptors to
         * setup the vector space for the cpu which comes online.
         * Prevent irq alloc/free across the bringup.
         */
        irq_lock_sparse();

        ret = cpuhp_bp_sync_alive(cpu);
        if (ret)
                goto out_unlock;

        ret = bringup_wait_for_ap_online(cpu);
        if (ret)
                goto out_unlock;

        irq_unlock_sparse();

        if (st->target <= CPUHP_AP_ONLINE_IDLE)
                return 0;

        return cpuhp_kick_ap(cpu, st, st->target);

out_unlock:
        irq_unlock_sparse();
        return ret;
}
#else
static int bringup_cpu(unsigned int cpu)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
        struct task_struct *idle = idle_thread_get(cpu);
        int ret;

        if (!cpuhp_can_boot_ap(cpu))
                return -EAGAIN;

        /*
         * Some architectures have to walk the irq descriptors to
         * setup the vector space for the cpu which comes online.
         *
         * Prevent irq alloc/free across the bringup by acquiring the
         * sparse irq lock. Hold it until the upcoming CPU completes the
         * startup in cpuhp_online_idle() which allows to avoid
         * intermediate synchronization points in the architecture code.
         */
        irq_lock_sparse();

        ret = __cpu_up(cpu, idle);
        if (ret)
                goto out_unlock;

        ret = cpuhp_bp_sync_alive(cpu);
        if (ret)
                goto out_unlock;

        ret = bringup_wait_for_ap_online(cpu);
        if (ret)
                goto out_unlock;

        irq_unlock_sparse();

        if (st->target <= CPUHP_AP_ONLINE_IDLE)
                return 0;

        return cpuhp_kick_ap(cpu, st, st->target);

out_unlock:
        irq_unlock_sparse();
        return ret;
}
#endif

static int finish_cpu(unsigned int cpu)
{
        struct task_struct *idle = idle_thread_get(cpu);
        struct mm_struct *mm = idle->active_mm;

        /*
         * sched_force_init_mm() ensured the use of &init_mm,
         * drop that refcount now that the CPU has stopped.
         */
        WARN_ON(mm != &init_mm);
        idle->active_mm = NULL;
        mmdrop_lazy_tlb(mm);

        return 0;
}

/*
 * Hotplug state machine related functions
 */

/*
 * Get the next state to run. Empty ones will be skipped. Returns true if a
 * state must be run.
 *
 * st->state will be modified ahead of time, to match state_to_run, as if it
 * has already ran.
 */
static bool cpuhp_next_state(bool bringup,
                             enum cpuhp_state *state_to_run,
                             struct cpuhp_cpu_state *st,
                             enum cpuhp_state target)
{
        do {
                if (bringup) {
                        if (st->state >= target)
                                return false;

                        *state_to_run = ++st->state;
                } else {
                        if (st->state <= target)
                                return false;

                        *state_to_run = st->state--;
                }

                if (!cpuhp_step_empty(bringup, cpuhp_get_step(*state_to_run)))
                        break;
        } while (true);

        return true;
}

static int __cpuhp_invoke_callback_range(bool bringup,
                                         unsigned int cpu,
                                         struct cpuhp_cpu_state *st,
                                         enum cpuhp_state target,
                                         bool nofail)
{
        enum cpuhp_state state;
        int ret = 0;

        while (cpuhp_next_state(bringup, &state, st, target)) {
                int err;

                err = cpuhp_invoke_callback(cpu, state, bringup, NULL, NULL);
                if (!err)
                        continue;

                if (nofail) {
                        pr_warn("CPU %u %s state %s (%d) failed (%d)\n",
                                cpu, bringup ? "UP" : "DOWN",
                                cpuhp_get_step(st->state)->name,
                                st->state, err);
                        ret = -1;
                } else {
                        ret = err;
                        break;
                }
        }

        return ret;
}

static inline int cpuhp_invoke_callback_range(bool bringup,
                                              unsigned int cpu,
                                              struct cpuhp_cpu_state *st,
                                              enum cpuhp_state target)
{
        return __cpuhp_invoke_callback_range(bringup, cpu, st, target, false);
}

static inline void cpuhp_invoke_callback_range_nofail(bool bringup,
                                                      unsigned int cpu,
                                                      struct cpuhp_cpu_state *st,
                                                      enum cpuhp_state target)
{
        __cpuhp_invoke_callback_range(bringup, cpu, st, target, true);
}

static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
{
        if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
                return true;
        /*
         * When CPU hotplug is disabled, then taking the CPU down is not
         * possible because takedown_cpu() and the architecture and
         * subsystem specific mechanisms are not available. So the CPU
         * which would be completely unplugged again needs to stay around
         * in the current state.
         */
        return st->state <= CPUHP_BRINGUP_CPU;
}

static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
                              enum cpuhp_state target)
{
        enum cpuhp_state prev_state = st->state;
        int ret = 0;

        ret = cpuhp_invoke_callback_range(true, cpu, st, target);
        if (ret) {
                pr_debug("CPU UP failed (%d) CPU %u state %s (%d)\n",
                         ret, cpu, cpuhp_get_step(st->state)->name,
                         st->state);

                cpuhp_reset_state(cpu, st, prev_state);
                if (can_rollback_cpu(st))
                        WARN_ON(cpuhp_invoke_callback_range(false, cpu, st,
                                                            prev_state));
        }
        return ret;
}

/*
 * The cpu hotplug threads manage the bringup and teardown of the cpus
 */
static int cpuhp_should_run(unsigned int cpu)
{
        struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);

        return st->should_run;
}

/*
 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
 * callbacks when a state gets [un]installed at runtime.
 *
 * Each invocation of this function by the smpboot thread does a single AP
 * state callback.
 *
 * It has 3 modes of operation:
 *  - single: runs st->cb_state
 *  - up:     runs ++st->state, while st->state < st->target
 *  - down:   runs st->state--, while st->state > st->target
 *
 * When complete or on error, should_run is cleared and the completion is fired.
 */
static void cpuhp_thread_fun(unsigned int cpu)
{
        struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
        bool bringup = st->bringup;
        enum cpuhp_state state;

        if (WARN_ON_ONCE(!st->should_run))
                return;

        /*
         * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
         * that if we see ->should_run we also see the rest of the state.
         */
        smp_mb();

        /*
         * The BP holds the hotplug lock, but we're now running on the AP,
         * ensure that anybody asserting the lock is held, will actually find
         * it so.
         */
        lockdep_acquire_cpus_lock();
        cpuhp_lock_acquire(bringup);

        if (st->single) {
                state = st->cb_state;
                st->should_run = false;
        } else {
                st->should_run = cpuhp_next_state(bringup, &state, st, st->target);
                if (!st->should_run)
                        goto end;
        }

        WARN_ON_ONCE(!cpuhp_is_ap_state(state));

        if (cpuhp_is_atomic_state(state)) {
                local_irq_disable();
                st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
                local_irq_enable();

                /*
                 * STARTING/DYING must not fail!
                 */
                WARN_ON_ONCE(st->result);
        } else {
                st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
        }

        if (st->result) {
                /*
                 * If we fail on a rollback, we're up a creek without no
                 * paddle, no way forward, no way back. We loose, thanks for
                 * playing.
                 */
                WARN_ON_ONCE(st->rollback);
                st->should_run = false;
        }

end:
        cpuhp_lock_release(bringup);
        lockdep_release_cpus_lock();

        if (!st->should_run)
                complete_ap_thread(st, bringup);
}

/* Invoke a single callback on a remote cpu */
static int
cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
                         struct hlist_node *node)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
        int ret;

        if (!cpu_online(cpu))
                return 0;

        cpuhp_lock_acquire(false);
        cpuhp_lock_release(false);

        cpuhp_lock_acquire(true);
        cpuhp_lock_release(true);

        /*
         * If we are up and running, use the hotplug thread. For early calls
         * we invoke the thread function directly.
         */
        if (!st->thread)
                return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);

        st->rollback = false;
        st->last = NULL;

        st->node = node;
        st->bringup = bringup;
        st->cb_state = state;
        st->single = true;

        __cpuhp_kick_ap(st);

        /*
         * If we failed and did a partial, do a rollback.
         */
        if ((ret = st->result) && st->last) {
                st->rollback = true;
                st->bringup = !bringup;

                __cpuhp_kick_ap(st);
        }

        /*
         * Clean up the leftovers so the next hotplug operation wont use stale
         * data.
         */
        st->node = st->last = NULL;
        return ret;
}

static int cpuhp_kick_ap_work(unsigned int cpu)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
        enum cpuhp_state prev_state = st->state;
        int ret;

        cpuhp_lock_acquire(false);
        cpuhp_lock_release(false);

        cpuhp_lock_acquire(true);
        cpuhp_lock_release(true);

        trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
        ret = cpuhp_kick_ap(cpu, st, st->target);
        trace_cpuhp_exit(cpu, st->state, prev_state, ret);

        return ret;
}

static struct smp_hotplug_thread cpuhp_threads = {
        .store                        = &cpuhp_state.thread,
        .thread_should_run        = cpuhp_should_run,
        .thread_fn                = cpuhp_thread_fun,
        .thread_comm                = "cpuhp/%u",
        .selfparking                = true,
};

static __init void cpuhp_init_state(void)
{
        struct cpuhp_cpu_state *st;
        int cpu;

        for_each_possible_cpu(cpu) {
                st = per_cpu_ptr(&cpuhp_state, cpu);
                init_completion(&st->done_up);
                init_completion(&st->done_down);
        }
}

void __init cpuhp_threads_init(void)
{
        cpuhp_init_state();
        BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
        kthread_unpark(this_cpu_read(cpuhp_state.thread));
}

#ifdef CONFIG_HOTPLUG_CPU
#ifndef arch_clear_mm_cpumask_cpu
#define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
#endif

/**
 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
 * @cpu: a CPU id
 *
 * This function walks all processes, finds a valid mm struct for each one and
 * then clears a corresponding bit in mm's cpumask.  While this all sounds
 * trivial, there are various non-obvious corner cases, which this function
 * tries to solve in a safe manner.
 *
 * Also note that the function uses a somewhat relaxed locking scheme, so it may
 * be called only for an already offlined CPU.
 */
void clear_tasks_mm_cpumask(int cpu)
{
        struct task_struct *p;

        /*
         * This function is called after the cpu is taken down and marked
         * offline, so its not like new tasks will ever get this cpu set in
         * their mm mask. -- Peter Zijlstra
         * Thus, we may use rcu_read_lock() here, instead of grabbing
         * full-fledged tasklist_lock.
         */
        WARN_ON(cpu_online(cpu));
        rcu_read_lock();
        for_each_process(p) {
                struct task_struct *t;

                /*
                 * Main thread might exit, but other threads may still have
                 * a valid mm. Find one.
                 */
                t = find_lock_task_mm(p);
                if (!t)
                        continue;
                arch_clear_mm_cpumask_cpu(cpu, t->mm);
                task_unlock(t);
        }
        rcu_read_unlock();
}

/* Take this CPU down. */
static int take_cpu_down(void *_param)
{
        struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
        enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
        int err, cpu = smp_processor_id();

        /* Ensure this CPU doesn't handle any more interrupts. */
        err = __cpu_disable();
        if (err < 0)
                return err;

        /*
         * Must be called from CPUHP_TEARDOWN_CPU, which means, as we are going
         * down, that the current state is CPUHP_TEARDOWN_CPU - 1.
         */
        WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));

        /*
         * Invoke the former CPU_DYING callbacks. DYING must not fail!
         */
        cpuhp_invoke_callback_range_nofail(false, cpu, st, target);

        /* Park the stopper thread */
        stop_machine_park(cpu);
        return 0;
}

static int takedown_cpu(unsigned int cpu)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
        int err;

        /* Park the smpboot threads */
        kthread_park(st->thread);

        /*
         * Prevent irq alloc/free while the dying cpu reorganizes the
         * interrupt affinities.
         */
        irq_lock_sparse();

        err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
        if (err) {
                /* CPU refused to die */
                irq_unlock_sparse();
                /* Unpark the hotplug thread so we can rollback there */
                kthread_unpark(st->thread);
                return err;
        }
        BUG_ON(cpu_online(cpu));

        /*
         * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
         * all runnable tasks from the CPU, there's only the idle task left now
         * that the migration thread is done doing the stop_machine thing.
         *
         * Wait for the stop thread to go away.
         */
        wait_for_ap_thread(st, false);
        BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);

        /* Interrupts are moved away from the dying cpu, reenable alloc/free */
        irq_unlock_sparse();

        hotplug_cpu__broadcast_tick_pull(cpu);
        /* This actually kills the CPU. */
        __cpu_die(cpu);

        cpuhp_bp_sync_dead(cpu);

        lockdep_cleanup_dead_cpu(cpu, idle_thread_get(cpu));

        /*
         * Callbacks must be re-integrated right away to the RCU state machine.
         * Otherwise an RCU callback could block a further teardown function
         * waiting for its completion.
         */
        rcutree_migrate_callbacks(cpu);

        return 0;
}

static void cpuhp_complete_idle_dead(void *arg)
{
        struct cpuhp_cpu_state *st = arg;

        complete_ap_thread(st, false);
}

void cpuhp_report_idle_dead(void)
{
        struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);

        BUG_ON(st->state != CPUHP_AP_OFFLINE);
        tick_assert_timekeeping_handover();
        rcutree_report_cpu_dead();
        st->state = CPUHP_AP_IDLE_DEAD;
        /*
         * We cannot call complete after rcutree_report_cpu_dead() so we delegate it
         * to an online cpu.
         */
        smp_call_function_single(cpumask_first(cpu_online_mask),
                                 cpuhp_complete_idle_dead, st, 0);
}

static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
                                enum cpuhp_state target)
{
        enum cpuhp_state prev_state = st->state;
        int ret = 0;

        ret = cpuhp_invoke_callback_range(false, cpu, st, target);
        if (ret) {
                pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n",
                         ret, cpu, cpuhp_get_step(st->state)->name,
                         st->state);

                cpuhp_reset_state(cpu, st, prev_state);

                if (st->state < prev_state)
                        WARN_ON(cpuhp_invoke_callback_range(true, cpu, st,
                                                            prev_state));
        }

        return ret;
}

/* Requires cpu_add_remove_lock to be held */
static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
                           enum cpuhp_state target)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
        int prev_state, ret = 0;

        if (num_online_cpus() == 1)
                return -EBUSY;

        if (!cpu_present(cpu))
                return -EINVAL;

        cpus_write_lock();

        /*
         * Keep at least one housekeeping cpu onlined to avoid generating
         * an empty sched_domain span.
         */
        if (cpumask_any_and(cpu_online_mask,
                            housekeeping_cpumask(HK_TYPE_DOMAIN)) >= nr_cpu_ids) {
                ret = -EBUSY;
                goto out;
        }

        cpuhp_tasks_frozen = tasks_frozen;

        prev_state = cpuhp_set_state(cpu, st, target);
        /*
         * If the current CPU state is in the range of the AP hotplug thread,
         * then we need to kick the thread.
         */
        if (st->state > CPUHP_TEARDOWN_CPU) {
                st->target = max((int)target, CPUHP_TEARDOWN_CPU);
                ret = cpuhp_kick_ap_work(cpu);
                /*
                 * The AP side has done the error rollback already. Just
                 * return the error code..
                 */
                if (ret)
                        goto out;

                /*
                 * We might have stopped still in the range of the AP hotplug
                 * thread. Nothing to do anymore.
                 */
                if (st->state > CPUHP_TEARDOWN_CPU)
                        goto out;

                st->target = target;
        }
        /*
         * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
         * to do the further cleanups.
         */
        ret = cpuhp_down_callbacks(cpu, st, target);
        if (ret && st->state < prev_state) {
                if (st->state == CPUHP_TEARDOWN_CPU) {
                        cpuhp_reset_state(cpu, st, prev_state);
                        __cpuhp_kick_ap(st);
                } else {
                        WARN(1, "DEAD callback error for CPU%d", cpu);
                }
        }

out:
        cpus_write_unlock();
        arch_smt_update();
        return ret;
}

static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
{
        /*
         * If the platform does not support hotplug, report it explicitly to
         * differentiate it from a transient offlining failure.
         */
        if (cpu_hotplug_offline_disabled)
                return -EOPNOTSUPP;
        if (cpu_hotplug_disabled)
                return -EBUSY;
        return _cpu_down(cpu, 0, target);
}

static int cpu_down(unsigned int cpu, enum cpuhp_state target)
{
        int err;

        cpu_maps_update_begin();
        err = cpu_down_maps_locked(cpu, target);
        cpu_maps_update_done();
        return err;
}

/**
 * cpu_device_down - Bring down a cpu device
 * @dev: Pointer to the cpu device to offline
 *
 * This function is meant to be used by device core cpu subsystem only.
 *
 * Other subsystems should use remove_cpu() instead.
 *
 * Return: %0 on success or a negative errno code
 */
int cpu_device_down(struct device *dev)
{
        return cpu_down(dev->id, CPUHP_OFFLINE);
}

int remove_cpu(unsigned int cpu)
{
        int ret;

        lock_device_hotplug();
        ret = device_offline(get_cpu_device(cpu));
        unlock_device_hotplug();

        return ret;
}
EXPORT_SYMBOL_GPL(remove_cpu);

void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
{
        unsigned int cpu;
        int error;

        cpu_maps_update_begin();

        /*
         * Make certain the cpu I'm about to reboot on is online.
         *
         * This is inline to what migrate_to_reboot_cpu() already do.
         */
        if (!cpu_online(primary_cpu))
                primary_cpu = cpumask_first(cpu_online_mask);

        for_each_online_cpu(cpu) {
                if (cpu == primary_cpu)
                        continue;

                error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
                if (error) {
                        pr_err("Failed to offline CPU%d - error=%d",
                                cpu, error);
                        break;
                }
        }

        /*
         * Ensure all but the reboot CPU are offline.
         */
        BUG_ON(num_online_cpus() > 1);

        /*
         * Make sure the CPUs won't be enabled by someone else after this
         * point. Kexec will reboot to a new kernel shortly resetting
         * everything along the way.
         */
        cpu_hotplug_disabled++;

        cpu_maps_update_done();
}

#else
#define takedown_cpu                NULL
#endif /*CONFIG_HOTPLUG_CPU*/

/**
 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
 * @cpu: cpu that just started
 *
 * It must be called by the arch code on the new cpu, before the new cpu
 * enables interrupts and before the "boot" cpu returns from __cpu_up().
 */
void notify_cpu_starting(unsigned int cpu)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
        enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);

        rcutree_report_cpu_starting(cpu);        /* Enables RCU usage on this CPU. */
        cpumask_set_cpu(cpu, &cpus_booted_once_mask);

        /*
         * STARTING must not fail!
         */
        cpuhp_invoke_callback_range_nofail(true, cpu, st, target);
}

/*
 * Called from the idle task. Wake up the controlling task which brings the
 * hotplug thread of the upcoming CPU up and then delegates the rest of the
 * online bringup to the hotplug thread.
 */
void cpuhp_online_idle(enum cpuhp_state state)
{
        struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);

        /* Happens for the boot cpu */
        if (state != CPUHP_AP_ONLINE_IDLE)
                return;

        cpuhp_ap_update_sync_state(SYNC_STATE_ONLINE);

        /*
         * Unpark the stopper thread before we start the idle loop (and start
         * scheduling); this ensures the stopper task is always available.
         */
        stop_machine_unpark(smp_processor_id());

        st->state = CPUHP_AP_ONLINE_IDLE;
        complete_ap_thread(st, true);
}

/* Requires cpu_add_remove_lock to be held */
static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
        struct task_struct *idle;
        int ret = 0;

        cpus_write_lock();

        if (!cpu_present(cpu)) {
                ret = -EINVAL;
                goto out;
        }

        /*
         * The caller of cpu_up() might have raced with another
         * caller. Nothing to do.
         */
        if (st->state >= target)
                goto out;

        if (st->state == CPUHP_OFFLINE) {
                /* Let it fail before we try to bring the cpu up */
                idle = idle_thread_get(cpu);
                if (IS_ERR(idle)) {
                        ret = PTR_ERR(idle);
                        goto out;
                }

                /*
                 * Reset stale stack state from the last time this CPU was online.
                 */
                scs_task_reset(idle);
                kasan_unpoison_task_stack(idle);
        }

        cpuhp_tasks_frozen = tasks_frozen;

        cpuhp_set_state(cpu, st, target);
        /*
         * If the current CPU state is in the range of the AP hotplug thread,
         * then we need to kick the thread once more.
         */
        if (st->state > CPUHP_BRINGUP_CPU) {
                ret = cpuhp_kick_ap_work(cpu);
                /*
                 * The AP side has done the error rollback already. Just
                 * return the error code..
                 */
                if (ret)
                        goto out;
        }

        /*
         * Try to reach the target state. We max out on the BP at
         * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
         * responsible for bringing it up to the target state.
         */
        target = min((int)target, CPUHP_BRINGUP_CPU);
        ret = cpuhp_up_callbacks(cpu, st, target);
out:
        cpus_write_unlock();
        arch_smt_update();
        return ret;
}

static int cpu_up(unsigned int cpu, enum cpuhp_state target)
{
        int err = 0;

        if (!cpu_possible(cpu)) {
                pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
                       cpu);
                return -EINVAL;
        }

        err = try_online_node(cpu_to_node(cpu));
        if (err)
                return err;

        cpu_maps_update_begin();

        if (cpu_hotplug_disabled) {
                err = -EBUSY;
                goto out;
        }
        if (!cpu_bootable(cpu)) {
                err = -EPERM;
                goto out;
        }

        err = _cpu_up(cpu, 0, target);
out:
        cpu_maps_update_done();
        return err;
}

/**
 * cpu_device_up - Bring up a cpu device
 * @dev: Pointer to the cpu device to online
 *
 * This function is meant to be used by device core cpu subsystem only.
 *
 * Other subsystems should use add_cpu() instead.
 *
 * Return: %0 on success or a negative errno code
 */
int cpu_device_up(struct device *dev)
{
        return cpu_up(dev->id, CPUHP_ONLINE);
}

int add_cpu(unsigned int cpu)
{
        int ret;

        lock_device_hotplug();
        ret = device_online(get_cpu_device(cpu));
        unlock_device_hotplug();

        return ret;
}
EXPORT_SYMBOL_GPL(add_cpu);

/**
 * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
 * @sleep_cpu: The cpu we hibernated on and should be brought up.
 *
 * On some architectures like arm64, we can hibernate on any CPU, but on
 * wake up the CPU we hibernated on might be offline as a side effect of
 * using maxcpus= for example.
 *
 * Return: %0 on success or a negative errno code
 */
int bringup_hibernate_cpu(unsigned int sleep_cpu)
{
        int ret;

        if (!cpu_online(sleep_cpu)) {
                pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
                ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
                if (ret) {
                        pr_err("Failed to bring hibernate-CPU up!\n");
                        return ret;
                }
        }
        return 0;
}

static void __init cpuhp_bringup_mask(const struct cpumask *mask, unsigned int ncpus,
                                      enum cpuhp_state target)
{
        unsigned int cpu;

        for_each_cpu(cpu, mask) {
                struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);

                if (cpu_up(cpu, target) && can_rollback_cpu(st)) {
                        /*
                         * If this failed then cpu_up() might have only
                         * rolled back to CPUHP_BP_KICK_AP for the final
                         * online. Clean it up. NOOP if already rolled back.
                         */
                        WARN_ON(cpuhp_invoke_callback_range(false, cpu, st, CPUHP_OFFLINE));
                }

                if (!--ncpus)
                        break;
        }
}

#ifdef CONFIG_HOTPLUG_PARALLEL
static bool __cpuhp_parallel_bringup __ro_after_init = true;

static int __init parallel_bringup_parse_param(char *arg)
{
        return kstrtobool(arg, &__cpuhp_parallel_bringup);
}
early_param("cpuhp.parallel", parallel_bringup_parse_param);

#ifdef CONFIG_HOTPLUG_SMT
static inline bool cpuhp_smt_aware(void)
{
        return cpu_smt_max_threads > 1;
}

static inline const struct cpumask *cpuhp_get_primary_thread_mask(void)
{
        return cpu_primary_thread_mask;
}
#else
static inline bool cpuhp_smt_aware(void)
{
        return false;
}
static inline const struct cpumask *cpuhp_get_primary_thread_mask(void)
{
        return cpu_none_mask;
}
#endif

bool __weak arch_cpuhp_init_parallel_bringup(void)
{
        return true;
}

/*
 * On architectures which have enabled parallel bringup this invokes all BP
 * prepare states for each of the to be onlined APs first. The last state
 * sends the startup IPI to the APs. The APs proceed through the low level
 * bringup code in parallel and then wait for the control CPU to release
 * them one by one for the final onlining procedure.
 *
 * This avoids waiting for each AP to respond to the startup IPI in
 * CPUHP_BRINGUP_CPU.
 */
static bool __init cpuhp_bringup_cpus_parallel(unsigned int ncpus)
{
        const struct cpumask *mask = cpu_present_mask;

        if (__cpuhp_parallel_bringup)
                __cpuhp_parallel_bringup = arch_cpuhp_init_parallel_bringup();
        if (!__cpuhp_parallel_bringup)
                return false;

        if (cpuhp_smt_aware()) {
                const struct cpumask *pmask = cpuhp_get_primary_thread_mask();
                static struct cpumask tmp_mask __initdata;

                /*
                 * X86 requires to prevent that SMT siblings stopped while
                 * the primary thread does a microcode update for various
                 * reasons. Bring the primary threads up first.
                 */
                cpumask_and(&tmp_mask, mask, pmask);
                cpuhp_bringup_mask(&tmp_mask, ncpus, CPUHP_BP_KICK_AP);
                cpuhp_bringup_mask(&tmp_mask, ncpus, CPUHP_ONLINE);
                /* Account for the online CPUs */
                ncpus -= num_online_cpus();
                if (!ncpus)
                        return true;
                /* Create the mask for secondary CPUs */
                cpumask_andnot(&tmp_mask, mask, pmask);
                mask = &tmp_mask;
        }

        /* Bring the not-yet started CPUs up */
        cpuhp_bringup_mask(mask, ncpus, CPUHP_BP_KICK_AP);
        cpuhp_bringup_mask(mask, ncpus, CPUHP_ONLINE);
        return true;
}
#else
static inline bool cpuhp_bringup_cpus_parallel(unsigned int ncpus) { return false; }
#endif /* CONFIG_HOTPLUG_PARALLEL */

void __init bringup_nonboot_cpus(unsigned int max_cpus)
{
        if (!max_cpus)
                return;

        /* Try parallel bringup optimization if enabled */
        if (cpuhp_bringup_cpus_parallel(max_cpus))
                return;

        /* Full per CPU serialized bringup */
        cpuhp_bringup_mask(cpu_present_mask, max_cpus, CPUHP_ONLINE);
}

#ifdef CONFIG_PM_SLEEP_SMP
static cpumask_var_t frozen_cpus;

int freeze_secondary_cpus(int primary)
{
        int cpu, error = 0;

        cpu_maps_update_begin();
        if (primary == -1) {
                primary = cpumask_first(cpu_online_mask);
                if (!housekeeping_cpu(primary, HK_TYPE_TIMER))
                        primary = housekeeping_any_cpu(HK_TYPE_TIMER);
        } else {
                if (!cpu_online(primary))
                        primary = cpumask_first(cpu_online_mask);
        }

        /*
         * We take down all of the non-boot CPUs in one shot to avoid races
         * with the userspace trying to use the CPU hotplug at the same time
         */
        cpumask_clear(frozen_cpus);

        pr_info("Disabling non-boot CPUs ...\n");
        for (cpu = nr_cpu_ids - 1; cpu >= 0; cpu--) {
                if (!cpu_online(cpu) || cpu == primary)
                        continue;

                if (pm_wakeup_pending()) {
                        pr_info("Wakeup pending. Abort CPU freeze\n");
                        error = -EBUSY;
                        break;
                }

                trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
                error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
                trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
                if (!error)
                        cpumask_set_cpu(cpu, frozen_cpus);
                else {
                        pr_err("Error taking CPU%d down: %d\n", cpu, error);
                        break;
                }
        }

        if (!error)
                BUG_ON(num_online_cpus() > 1);
        else
                pr_err("Non-boot CPUs are not disabled\n");

        /*
         * Make sure the CPUs won't be enabled by someone else. We need to do
         * this even in case of failure as all freeze_secondary_cpus() users are
         * supposed to do thaw_secondary_cpus() on the failure path.
         */
        cpu_hotplug_disabled++;

        cpu_maps_update_done();
        return error;
}

void __weak arch_thaw_secondary_cpus_begin(void)
{
}

void __weak arch_thaw_secondary_cpus_end(void)
{
}

void thaw_secondary_cpus(void)
{
        int cpu, error;

        /* Allow everyone to use the CPU hotplug again */
        cpu_maps_update_begin();
        __cpu_hotplug_enable();
        if (cpumask_empty(frozen_cpus))
                goto out;

        pr_info("Enabling non-boot CPUs ...\n");

        arch_thaw_secondary_cpus_begin();

        for_each_cpu(cpu, frozen_cpus) {
                trace_suspend_resume(TPS("CPU_ON"), cpu, true);
                error = _cpu_up(cpu, 1, CPUHP_ONLINE);
                trace_suspend_resume(TPS("CPU_ON"), cpu, false);
                if (!error) {
                        pr_info("CPU%d is up\n", cpu);
                        continue;
                }
                pr_warn("Error taking CPU%d up: %d\n", cpu, error);
        }

        arch_thaw_secondary_cpus_end();

        cpumask_clear(frozen_cpus);
out:
        cpu_maps_update_done();
}

static int __init alloc_frozen_cpus(void)
{
        if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
                return -ENOMEM;
        return 0;
}
core_initcall(alloc_frozen_cpus);

/*
 * When callbacks for CPU hotplug notifications are being executed, we must
 * ensure that the state of the system with respect to the tasks being frozen
 * or not, as reported by the notification, remains unchanged *throughout the
 * duration* of the execution of the callbacks.
 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
 *
 * This synchronization is implemented by mutually excluding regular CPU
 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
 * Hibernate notifications.
 */
static int
cpu_hotplug_pm_callback(struct notifier_block *nb,
                        unsigned long action, void *ptr)
{
        switch (action) {

        case PM_SUSPEND_PREPARE:
        case PM_HIBERNATION_PREPARE:
                cpu_hotplug_disable();
                break;

        case PM_POST_SUSPEND:
        case PM_POST_HIBERNATION:
                cpu_hotplug_enable();
                break;

        default:
                return NOTIFY_DONE;
        }

        return NOTIFY_OK;
}


static int __init cpu_hotplug_pm_sync_init(void)
{
        /*
         * cpu_hotplug_pm_callback has higher priority than x86
         * bsp_pm_callback which depends on cpu_hotplug_pm_callback
         * to disable cpu hotplug to avoid cpu hotplug race.
         */
        pm_notifier(cpu_hotplug_pm_callback, 0);
        return 0;
}
core_initcall(cpu_hotplug_pm_sync_init);

#endif /* CONFIG_PM_SLEEP_SMP */

int __boot_cpu_id;

#endif /* CONFIG_SMP */

/* Boot processor state steps */
static struct cpuhp_step cpuhp_hp_states[] = {
        [CPUHP_OFFLINE] = {
                .name                        = "offline",
                .startup.single                = NULL,
                .teardown.single        = NULL,
        },
#ifdef CONFIG_SMP
        [CPUHP_CREATE_THREADS]= {
                .name                        = "threads:prepare",
                .startup.single                = smpboot_create_threads,
                .teardown.single        = NULL,
                .cant_stop                = true,
        },
        [CPUHP_RANDOM_PREPARE] = {
                .name                        = "random:prepare",
                .startup.single                = random_prepare_cpu,
                .teardown.single        = NULL,
        },
        [CPUHP_WORKQUEUE_PREP] = {
                .name                        = "workqueue:prepare",
                .startup.single                = workqueue_prepare_cpu,
                .teardown.single        = NULL,
        },
        [CPUHP_HRTIMERS_PREPARE] = {
                .name                        = "hrtimers:prepare",
                .startup.single                = hrtimers_prepare_cpu,
                .teardown.single        = NULL,
        },
        [CPUHP_SMPCFD_PREPARE] = {
                .name                        = "smpcfd:prepare",
                .startup.single                = smpcfd_prepare_cpu,
                .teardown.single        = smpcfd_dead_cpu,
        },
        [CPUHP_RELAY_PREPARE] = {
                .name                        = "relay:prepare",
                .startup.single                = relay_prepare_cpu,
                .teardown.single        = NULL,
        },
        [CPUHP_RCUTREE_PREP] = {
                .name                        = "RCU/tree:prepare",
                .startup.single                = rcutree_prepare_cpu,
                .teardown.single        = rcutree_dead_cpu,
        },
        /*
         * On the tear-down path, timers_dead_cpu() must be invoked
         * before blk_mq_queue_reinit_notify() from notify_dead(),
         * otherwise a RCU stall occurs.
         */
        [CPUHP_TIMERS_PREPARE] = {
                .name                        = "timers:prepare",
                .startup.single                = timers_prepare_cpu,
                .teardown.single        = timers_dead_cpu,
        },

#ifdef CONFIG_HOTPLUG_SPLIT_STARTUP
        /*
         * Kicks the AP alive. AP will wait in cpuhp_ap_sync_alive() until
         * the next step will release it.
         */
        [CPUHP_BP_KICK_AP] = {
                .name                        = "cpu:kick_ap",
                .startup.single                = cpuhp_kick_ap_alive,
        },

        /*
         * Waits for the AP to reach cpuhp_ap_sync_alive() and then
         * releases it for the complete bringup.
         */
        [CPUHP_BRINGUP_CPU] = {
                .name                        = "cpu:bringup",
                .startup.single                = cpuhp_bringup_ap,
                .teardown.single        = finish_cpu,
                .cant_stop                = true,
        },
#else
        /*
         * All-in-one CPU bringup state which includes the kick alive.
         */
        [CPUHP_BRINGUP_CPU] = {
                .name                        = "cpu:bringup",
                .startup.single                = bringup_cpu,
                .teardown.single        = finish_cpu,
                .cant_stop                = true,
        },
#endif
        /* Final state before CPU kills itself */
        [CPUHP_AP_IDLE_DEAD] = {
                .name                        = "idle:dead",
        },
        /*
         * Last state before CPU enters the idle loop to die. Transient state
         * for synchronization.
         */
        [CPUHP_AP_OFFLINE] = {
                .name                        = "ap:offline",
                .cant_stop                = true,
        },
        /* First state is scheduler control. Interrupts are disabled */
        [CPUHP_AP_SCHED_STARTING] = {
                .name                        = "sched:starting",
                .startup.single                = sched_cpu_starting,
                .teardown.single        = sched_cpu_dying,
        },
        [CPUHP_AP_RCUTREE_DYING] = {
                .name                        = "RCU/tree:dying",
                .startup.single                = NULL,
                .teardown.single        = rcutree_dying_cpu,
        },
        [CPUHP_AP_SMPCFD_DYING] = {
                .name                        = "smpcfd:dying",
                .startup.single                = NULL,
                .teardown.single        = smpcfd_dying_cpu,
        },
        [CPUHP_AP_HRTIMERS_DYING] = {
                .name                        = "hrtimers:dying",
                .startup.single                = hrtimers_cpu_starting,
                .teardown.single        = hrtimers_cpu_dying,
        },
        [CPUHP_AP_TICK_DYING] = {
                .name                        = "tick:dying",
                .startup.single                = NULL,
                .teardown.single        = tick_cpu_dying,
        },
        /* Entry state on starting. Interrupts enabled from here on. Transient
         * state for synchronsization */
        [CPUHP_AP_ONLINE] = {
                .name                        = "ap:online",
        },
        /*
         * Handled on control processor until the plugged processor manages
         * this itself.
         */
        [CPUHP_TEARDOWN_CPU] = {
                .name                        = "cpu:teardown",
                .startup.single                = NULL,
                .teardown.single        = takedown_cpu,
                .cant_stop                = true,
        },

        [CPUHP_AP_SCHED_WAIT_EMPTY] = {
                .name                        = "sched:waitempty",
                .startup.single                = NULL,
                .teardown.single        = sched_cpu_wait_empty,
        },

        /* Handle smpboot threads park/unpark */
        [CPUHP_AP_SMPBOOT_THREADS] = {
                .name                        = "smpboot/threads:online",
                .startup.single                = smpboot_unpark_threads,
                .teardown.single        = smpboot_park_threads,
        },
        [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
                .name                        = "irq/affinity:online",
                .startup.single                = irq_affinity_online_cpu,
                .teardown.single        = NULL,
        },
        [CPUHP_AP_PERF_ONLINE] = {
                .name                        = "perf:online",
                .startup.single                = perf_event_init_cpu,
                .teardown.single        = perf_event_exit_cpu,
        },
        [CPUHP_AP_WATCHDOG_ONLINE] = {
                .name                        = "lockup_detector:online",
                .startup.single                = lockup_detector_online_cpu,
                .teardown.single        = lockup_detector_offline_cpu,
        },
        [CPUHP_AP_WORKQUEUE_ONLINE] = {
                .name                        = "workqueue:online",
                .startup.single                = workqueue_online_cpu,
                .teardown.single        = workqueue_offline_cpu,
        },
        [CPUHP_AP_RANDOM_ONLINE] = {
                .name                        = "random:online",
                .startup.single                = random_online_cpu,
                .teardown.single        = NULL,
        },
        [CPUHP_AP_RCUTREE_ONLINE] = {
                .name                        = "RCU/tree:online",
                .startup.single                = rcutree_online_cpu,
                .teardown.single        = rcutree_offline_cpu,
        },
#endif
        /*
         * The dynamically registered state space is here
         */

#ifdef CONFIG_SMP
        /* Last state is scheduler control setting the cpu active */
        [CPUHP_AP_ACTIVE] = {
                .name                        = "sched:active",
                .startup.single                = sched_cpu_activate,
                .teardown.single        = sched_cpu_deactivate,
        },
#endif

        /* CPU is fully up and running. */
        [CPUHP_ONLINE] = {
                .name                        = "online",
                .startup.single                = NULL,
                .teardown.single        = NULL,
        },
};

/* Sanity check for callbacks */
static int cpuhp_cb_check(enum cpuhp_state state)
{
        if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
                return -EINVAL;
        return 0;
}

/*
 * Returns a free for dynamic slot assignment of the Online state. The states
 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
 * by having no name assigned.
 */
static int cpuhp_reserve_state(enum cpuhp_state state)
{
        enum cpuhp_state i, end;
        struct cpuhp_step *step;

        switch (state) {
        case CPUHP_AP_ONLINE_DYN:
                step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
                end = CPUHP_AP_ONLINE_DYN_END;
                break;
        case CPUHP_BP_PREPARE_DYN:
                step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
                end = CPUHP_BP_PREPARE_DYN_END;
                break;
        default:
                return -EINVAL;
        }

        for (i = state; i <= end; i++, step++) {
                if (!step->name)
                        return i;
        }
        WARN(1, "No more dynamic states available for CPU hotplug\n");
        return -ENOSPC;
}

static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
                                 int (*startup)(unsigned int cpu),
                                 int (*teardown)(unsigned int cpu),
                                 bool multi_instance)
{
        /* (Un)Install the callbacks for further cpu hotplug operations */
        struct cpuhp_step *sp;
        int ret = 0;

        /*
         * If name is NULL, then the state gets removed.
         *
         * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
         * the first allocation from these dynamic ranges, so the removal
         * would trigger a new allocation and clear the wrong (already
         * empty) state, leaving the callbacks of the to be cleared state
         * dangling, which causes wreckage on the next hotplug operation.
         */
        if (name && (state == CPUHP_AP_ONLINE_DYN ||
                     state == CPUHP_BP_PREPARE_DYN)) {
                ret = cpuhp_reserve_state(state);
                if (ret < 0)
                        return ret;
                state = ret;
        }
        sp = cpuhp_get_step(state);
        if (name && sp->name)
                return -EBUSY;

        sp->startup.single = startup;
        sp->teardown.single = teardown;
        sp->name = name;
        sp->multi_instance = multi_instance;
        INIT_HLIST_HEAD(&sp->list);
        return ret;
}

static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
{
        return cpuhp_get_step(state)->teardown.single;
}

/*
 * Call the startup/teardown function for a step either on the AP or
 * on the current CPU.
 */
static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
                            struct hlist_node *node)
{
        struct cpuhp_step *sp = cpuhp_get_step(state);
        int ret;

        /*
         * If there's nothing to do, we done.
         * Relies on the union for multi_instance.
         */
        if (cpuhp_step_empty(bringup, sp))
                return 0;
        /*
         * The non AP bound callbacks can fail on bringup. On teardown
         * e.g. module removal we crash for now.
         */
#ifdef CONFIG_SMP
        if (cpuhp_is_ap_state(state))
                ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
        else
                ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
#else
        if (cpuhp_is_atomic_state(state)) {
                guard(irqsave)();
                ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
                /* STARTING/DYING must not fail! */
                WARN_ON_ONCE(ret);
        } else {
                ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
        }
#endif
        BUG_ON(ret && !bringup);
        return ret;
}

/*
 * Called from __cpuhp_setup_state on a recoverable failure.
 *
 * Note: The teardown callbacks for rollback are not allowed to fail!
 */
static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
                                   struct hlist_node *node)
{
        int cpu;

        /* Roll back the already executed steps on the other cpus */
        for_each_present_cpu(cpu) {
                struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
                int cpustate = st->state;

                if (cpu >= failedcpu)
                        break;

                /* Did we invoke the startup call on that cpu ? */
                if (cpustate >= state)
                        cpuhp_issue_call(cpu, state, false, node);
        }
}

int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
                                          struct hlist_node *node,
                                          bool invoke)
{
        struct cpuhp_step *sp;
        int cpu;
        int ret;

        lockdep_assert_cpus_held();

        sp = cpuhp_get_step(state);
        if (sp->multi_instance == false)
                return -EINVAL;

        mutex_lock(&cpuhp_state_mutex);

        if (!invoke || !sp->startup.multi)
                goto add_node;

        /*
         * Try to call the startup callback for each present cpu
         * depending on the hotplug state of the cpu.
         */
        for_each_present_cpu(cpu) {
                struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
                int cpustate = st->state;

                if (cpustate < state)
                        continue;

                ret = cpuhp_issue_call(cpu, state, true, node);
                if (ret) {
                        if (sp->teardown.multi)
                                cpuhp_rollback_install(cpu, state, node);
                        goto unlock;
                }
        }
add_node:
        ret = 0;
        hlist_add_head(node, &sp->list);
unlock:
        mutex_unlock(&cpuhp_state_mutex);
        return ret;
}

int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
                               bool invoke)
{
        int ret;

        cpus_read_lock();
        ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
        cpus_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);

/**
 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
 * @state:                The state to setup
 * @name:                Name of the step
 * @invoke:                If true, the startup function is invoked for cpus where
 *                        cpu state >= @state
 * @startup:                startup callback function
 * @teardown:                teardown callback function
 * @multi_instance:        State is set up for multiple instances which get
 *                        added afterwards.
 *
 * The caller needs to hold cpus read locked while calling this function.
 * Return:
 *   On success:
 *      Positive state number if @state is CPUHP_AP_ONLINE_DYN or CPUHP_BP_PREPARE_DYN;
 *      0 for all other states
 *   On failure: proper (negative) error code
 */
int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
                                   const char *name, bool invoke,
                                   int (*startup)(unsigned int cpu),
                                   int (*teardown)(unsigned int cpu),
                                   bool multi_instance)
{
        int cpu, ret = 0;
        bool dynstate;

        lockdep_assert_cpus_held();

        if (cpuhp_cb_check(state) || !name)
                return -EINVAL;

        mutex_lock(&cpuhp_state_mutex);

        ret = cpuhp_store_callbacks(state, name, startup, teardown,
                                    multi_instance);

        dynstate = state == CPUHP_AP_ONLINE_DYN || state == CPUHP_BP_PREPARE_DYN;
        if (ret > 0 && dynstate) {
                state = ret;
                ret = 0;
        }

        if (ret || !invoke || !startup)
                goto out;

        /*
         * Try to call the startup callback for each present cpu
         * depending on the hotplug state of the cpu.
         */
        for_each_present_cpu(cpu) {
                struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
                int cpustate = st->state;

                if (cpustate < state)
                        continue;

                ret = cpuhp_issue_call(cpu, state, true, NULL);
                if (ret) {
                        if (teardown)
                                cpuhp_rollback_install(cpu, state, NULL);
                        cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
                        goto out;
                }
        }
out:
        mutex_unlock(&cpuhp_state_mutex);
        /*
         * If the requested state is CPUHP_AP_ONLINE_DYN or CPUHP_BP_PREPARE_DYN,
         * return the dynamically allocated state in case of success.
         */
        if (!ret && dynstate)
                return state;
        return ret;
}
EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);

int __cpuhp_setup_state(enum cpuhp_state state,
                        const char *name, bool invoke,
                        int (*startup)(unsigned int cpu),
                        int (*teardown)(unsigned int cpu),
                        bool multi_instance)
{
        int ret;

        cpus_read_lock();
        ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
                                             teardown, multi_instance);
        cpus_read_unlock();
        return ret;
}
EXPORT_SYMBOL(__cpuhp_setup_state);

int __cpuhp_state_remove_instance(enum cpuhp_state state,
                                  struct hlist_node *node, bool invoke)
{
        struct cpuhp_step *sp = cpuhp_get_step(state);
        int cpu;

        BUG_ON(cpuhp_cb_check(state));

        if (!sp->multi_instance)
                return -EINVAL;

        cpus_read_lock();
        mutex_lock(&cpuhp_state_mutex);

        if (!invoke || !cpuhp_get_teardown_cb(state))
                goto remove;
        /*
         * Call the teardown callback for each present cpu depending
         * on the hotplug state of the cpu. This function is not
         * allowed to fail currently!
         */
        for_each_present_cpu(cpu) {
                struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
                int cpustate = st->state;

                if (cpustate >= state)
                        cpuhp_issue_call(cpu, state, false, node);
        }

remove:
        hlist_del(node);
        mutex_unlock(&cpuhp_state_mutex);
        cpus_read_unlock();

        return 0;
}
EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);

/**
 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
 * @state:        The state to remove
 * @invoke:        If true, the teardown function is invoked for cpus where
 *                cpu state >= @state
 *
 * The caller needs to hold cpus read locked while calling this function.
 * The teardown callback is currently not allowed to fail. Think
 * about module removal!
 */
void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
{
        struct cpuhp_step *sp = cpuhp_get_step(state);
        int cpu;

        BUG_ON(cpuhp_cb_check(state));

        lockdep_assert_cpus_held();

        mutex_lock(&cpuhp_state_mutex);
        if (sp->multi_instance) {
                WARN(!hlist_empty(&sp->list),
                     "Error: Removing state %d which has instances left.\n",
                     state);
                goto remove;
        }

        if (!invoke || !cpuhp_get_teardown_cb(state))
                goto remove;

        /*
         * Call the teardown callback for each present cpu depending
         * on the hotplug state of the cpu. This function is not
         * allowed to fail currently!
         */
        for_each_present_cpu(cpu) {
                struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
                int cpustate = st->state;

                if (cpustate >= state)
                        cpuhp_issue_call(cpu, state, false, NULL);
        }
remove:
        cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
        mutex_unlock(&cpuhp_state_mutex);
}
EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);

void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
{
        cpus_read_lock();
        __cpuhp_remove_state_cpuslocked(state, invoke);
        cpus_read_unlock();
}
EXPORT_SYMBOL(__cpuhp_remove_state);

#ifdef CONFIG_HOTPLUG_SMT
static void cpuhp_offline_cpu_device(unsigned int cpu)
{
        struct device *dev = get_cpu_device(cpu);

        dev_set_offline(dev);
        /* Tell user space about the state change */
        kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
}

static void cpuhp_online_cpu_device(unsigned int cpu)
{
        struct device *dev = get_cpu_device(cpu);

        dev_clear_offline(dev);
        /* Tell user space about the state change */
        kobject_uevent(&dev->kobj, KOBJ_ONLINE);
}

int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
{
        int cpu, ret = 0;

        cpu_maps_update_begin();
        for_each_online_cpu(cpu) {
                if (topology_is_primary_thread(cpu))
                        continue;
                /*
                 * Disable can be called with CPU_SMT_ENABLED when changing
                 * from a higher to lower number of SMT threads per core.
                 */
                if (ctrlval == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu))
                        continue;
                ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
                if (ret)
                        break;
                /*
                 * As this needs to hold the cpu maps lock it's impossible
                 * to call device_offline() because that ends up calling
                 * cpu_down() which takes cpu maps lock. cpu maps lock
                 * needs to be held as this might race against in kernel
                 * abusers of the hotplug machinery (thermal management).
                 *
                 * So nothing would update device:offline state. That would
                 * leave the sysfs entry stale and prevent onlining after
                 * smt control has been changed to 'off' again. This is
                 * called under the sysfs hotplug lock, so it is properly
                 * serialized against the regular offline usage.
                 */
                cpuhp_offline_cpu_device(cpu);
        }
        if (!ret)
                cpu_smt_control = ctrlval;
        cpu_maps_update_done();
        return ret;
}

/* Check if the core a CPU belongs to is online */
#if !defined(topology_is_core_online)
static inline bool topology_is_core_online(unsigned int cpu)
{
        return true;
}
#endif

int cpuhp_smt_enable(void)
{
        int cpu, ret = 0;

        cpu_maps_update_begin();
        cpu_smt_control = CPU_SMT_ENABLED;
        for_each_present_cpu(cpu) {
                /* Skip online CPUs and CPUs on offline nodes */
                if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
                        continue;
                if (!cpu_smt_thread_allowed(cpu) || !topology_is_core_online(cpu))
                        continue;
                ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
                if (ret)
                        break;
                /* See comment in cpuhp_smt_disable() */
                cpuhp_online_cpu_device(cpu);
        }
        cpu_maps_update_done();
        return ret;
}
#endif

#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
static ssize_t state_show(struct device *dev,
                          struct device_attribute *attr, char *buf)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);

        return sprintf(buf, "%d\n", st->state);
}
static DEVICE_ATTR_RO(state);

static ssize_t target_store(struct device *dev, struct device_attribute *attr,
                            const char *buf, size_t count)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
        struct cpuhp_step *sp;
        int target, ret;

        ret = kstrtoint(buf, 10, &target);
        if (ret)
                return ret;

#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
        if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
                return -EINVAL;
#else
        if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
                return -EINVAL;
#endif

        ret = lock_device_hotplug_sysfs();
        if (ret)
                return ret;

        mutex_lock(&cpuhp_state_mutex);
        sp = cpuhp_get_step(target);
        ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
        mutex_unlock(&cpuhp_state_mutex);
        if (ret)
                goto out;

        if (st->state < target)
                ret = cpu_up(dev->id, target);
        else if (st->state > target)
                ret = cpu_down(dev->id, target);
        else if (WARN_ON(st->target != target))
                st->target = target;
out:
        unlock_device_hotplug();
        return ret ? ret : count;
}

static ssize_t target_show(struct device *dev,
                           struct device_attribute *attr, char *buf)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);

        return sprintf(buf, "%d\n", st->target);
}
static DEVICE_ATTR_RW(target);

static ssize_t fail_store(struct device *dev, struct device_attribute *attr,
                          const char *buf, size_t count)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
        struct cpuhp_step *sp;
        int fail, ret;

        ret = kstrtoint(buf, 10, &fail);
        if (ret)
                return ret;

        if (fail == CPUHP_INVALID) {
                st->fail = fail;
                return count;
        }

        if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
                return -EINVAL;

        /*
         * Cannot fail STARTING/DYING callbacks.
         */
        if (cpuhp_is_atomic_state(fail))
                return -EINVAL;

        /*
         * DEAD callbacks cannot fail...
         * ... neither can CPUHP_BRINGUP_CPU during hotunplug. The latter
         * triggering STARTING callbacks, a failure in this state would
         * hinder rollback.
         */
        if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)
                return -EINVAL;

        /*
         * Cannot fail anything that doesn't have callbacks.
         */
        mutex_lock(&cpuhp_state_mutex);
        sp = cpuhp_get_step(fail);
        if (!sp->startup.single && !sp->teardown.single)
                ret = -EINVAL;
        mutex_unlock(&cpuhp_state_mutex);
        if (ret)
                return ret;

        st->fail = fail;

        return count;
}

static ssize_t fail_show(struct device *dev,
                         struct device_attribute *attr, char *buf)
{
        struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);

        return sprintf(buf, "%d\n", st->fail);
}

static DEVICE_ATTR_RW(fail);

static struct attribute *cpuhp_cpu_attrs[] = {
        &dev_attr_state.attr,
        &dev_attr_target.attr,
        &dev_attr_fail.attr,
        NULL
};

static const struct attribute_group cpuhp_cpu_attr_group = {
        .attrs = cpuhp_cpu_attrs,
        .name = "hotplug",
};

static ssize_t states_show(struct device *dev, struct device_attribute *attr, char *buf)
{
        ssize_t res = 0;
        int i;

        mutex_lock(&cpuhp_state_mutex);
        for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
                struct cpuhp_step *sp = cpuhp_get_step(i);

                if (sp->name)
                        res += sysfs_emit_at(buf, res, "%3d: %s\n", i, sp->name);
        }
        mutex_unlock(&cpuhp_state_mutex);
        return res;
}
static DEVICE_ATTR_RO(states);

static struct attribute *cpuhp_cpu_root_attrs[] = {
        &dev_attr_states.attr,
        NULL
};

static const struct attribute_group cpuhp_cpu_root_attr_group = {
        .attrs = cpuhp_cpu_root_attrs,
        .name = "hotplug",
};

#ifdef CONFIG_HOTPLUG_SMT

static bool cpu_smt_num_threads_valid(unsigned int threads)
{
        if (IS_ENABLED(CONFIG_SMT_NUM_THREADS_DYNAMIC))
                return threads >= 1 && threads <= cpu_smt_max_threads;
        return threads == 1 || threads == cpu_smt_max_threads;
}

static ssize_t
__store_smt_control(struct device *dev, struct device_attribute *attr,
                    const char *buf, size_t count)
{
        int ctrlval, ret, num_threads, orig_threads;
        bool force_off;

        if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
                return -EPERM;

        if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
                return -ENODEV;

        if (sysfs_streq(buf, "on")) {
                ctrlval = CPU_SMT_ENABLED;
                num_threads = cpu_smt_max_threads;
        } else if (sysfs_streq(buf, "off")) {
                ctrlval = CPU_SMT_DISABLED;
                num_threads = 1;
        } else if (sysfs_streq(buf, "forceoff")) {
                ctrlval = CPU_SMT_FORCE_DISABLED;
                num_threads = 1;
        } else if (kstrtoint(buf, 10, &num_threads) == 0) {
                if (num_threads == 1)
                        ctrlval = CPU_SMT_DISABLED;
                else if (cpu_smt_num_threads_valid(num_threads))
                        ctrlval = CPU_SMT_ENABLED;
                else
                        return -EINVAL;
        } else {
                return -EINVAL;
        }

        ret = lock_device_hotplug_sysfs();
        if (ret)
                return ret;

        orig_threads = cpu_smt_num_threads;
        cpu_smt_num_threads = num_threads;

        force_off = ctrlval != cpu_smt_control && ctrlval == CPU_SMT_FORCE_DISABLED;

        if (num_threads > orig_threads)
                ret = cpuhp_smt_enable();
        else if (num_threads < orig_threads || force_off)
                ret = cpuhp_smt_disable(ctrlval);

        unlock_device_hotplug();
        return ret ? ret : count;
}

#else /* !CONFIG_HOTPLUG_SMT */
static ssize_t
__store_smt_control(struct device *dev, struct device_attribute *attr,
                    const char *buf, size_t count)
{
        return -ENODEV;
}
#endif /* CONFIG_HOTPLUG_SMT */

static const char *smt_states[] = {
        [CPU_SMT_ENABLED]                = "on",
        [CPU_SMT_DISABLED]                = "off",
        [CPU_SMT_FORCE_DISABLED]        = "forceoff",
        [CPU_SMT_NOT_SUPPORTED]                = "notsupported",
        [CPU_SMT_NOT_IMPLEMENTED]        = "notimplemented",
};

static ssize_t control_show(struct device *dev,
                            struct device_attribute *attr, char *buf)
{
        const char *state = smt_states[cpu_smt_control];

#ifdef CONFIG_HOTPLUG_SMT
        /*
         * If SMT is enabled but not all threads are enabled then show the
         * number of threads. If all threads are enabled show "on". Otherwise
         * show the state name.
         */
        if (cpu_smt_control == CPU_SMT_ENABLED &&
            cpu_smt_num_threads != cpu_smt_max_threads)
                return sysfs_emit(buf, "%d\n", cpu_smt_num_threads);
#endif

        return sysfs_emit(buf, "%s\n", state);
}

static ssize_t control_store(struct device *dev, struct device_attribute *attr,
                             const char *buf, size_t count)
{
        return __store_smt_control(dev, attr, buf, count);
}
static DEVICE_ATTR_RW(control);

static ssize_t active_show(struct device *dev,
                           struct device_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%d\n", sched_smt_active());
}
static DEVICE_ATTR_RO(active);

static struct attribute *cpuhp_smt_attrs[] = {
        &dev_attr_control.attr,
        &dev_attr_active.attr,
        NULL
};

static const struct attribute_group cpuhp_smt_attr_group = {
        .attrs = cpuhp_smt_attrs,
        .name = "smt",
};

static int __init cpu_smt_sysfs_init(void)
{
        struct device *dev_root;
        int ret = -ENODEV;

        dev_root = bus_get_dev_root(&cpu_subsys);
        if (dev_root) {
                ret = sysfs_create_group(&dev_root->kobj, &cpuhp_smt_attr_group);
                put_device(dev_root);
        }
        return ret;
}

static int __init cpuhp_sysfs_init(void)
{
        struct device *dev_root;
        int cpu, ret;

        ret = cpu_smt_sysfs_init();
        if (ret)
                return ret;

        dev_root = bus_get_dev_root(&cpu_subsys);
        if (dev_root) {
                ret = sysfs_create_group(&dev_root->kobj, &cpuhp_cpu_root_attr_group);
                put_device(dev_root);
                if (ret)
                        return ret;
        }

        for_each_possible_cpu(cpu) {
                struct device *dev = get_cpu_device(cpu);

                if (!dev)
                        continue;
                ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
                if (ret)
                        return ret;
        }
        return 0;
}
device_initcall(cpuhp_sysfs_init);
#endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */

/*
 * cpu_bit_bitmap[] is a special, "compressed" data structure that
 * represents all NR_CPUS bits binary values of 1<<nr.
 *
 * It is used by cpumask_of() to get a constant address to a CPU
 * mask value that has a single bit set only.
 */

/* cpu_bit_bitmap[0] is empty - so we can back into it */
#define MASK_DECLARE_1(x)        [x+1][0] = (1UL << (x))
#define MASK_DECLARE_2(x)        MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
#define MASK_DECLARE_4(x)        MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
#define MASK_DECLARE_8(x)        MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)

const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {

        MASK_DECLARE_8(0),        MASK_DECLARE_8(8),
        MASK_DECLARE_8(16),        MASK_DECLARE_8(24),
#if BITS_PER_LONG > 32
        MASK_DECLARE_8(32),        MASK_DECLARE_8(40),
        MASK_DECLARE_8(48),        MASK_DECLARE_8(56),
#endif
};
EXPORT_SYMBOL_GPL(cpu_bit_bitmap);

const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
EXPORT_SYMBOL(cpu_all_bits);

#ifdef CONFIG_INIT_ALL_POSSIBLE
struct cpumask __cpu_possible_mask __ro_after_init
        = {CPU_BITS_ALL};
unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
#else
struct cpumask __cpu_possible_mask __ro_after_init;
unsigned int __num_possible_cpus __ro_after_init;
#endif
EXPORT_SYMBOL(__cpu_possible_mask);
EXPORT_SYMBOL(__num_possible_cpus);

struct cpumask __cpu_online_mask __read_mostly;
EXPORT_SYMBOL(__cpu_online_mask);

struct cpumask __cpu_enabled_mask __read_mostly;
EXPORT_SYMBOL(__cpu_enabled_mask);

struct cpumask __cpu_present_mask __read_mostly;
EXPORT_SYMBOL(__cpu_present_mask);

struct cpumask __cpu_active_mask __read_mostly;
EXPORT_SYMBOL(__cpu_active_mask);

struct cpumask __cpu_dying_mask __read_mostly;
EXPORT_SYMBOL(__cpu_dying_mask);

atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);

void init_cpu_present(const struct cpumask *src)
{
        cpumask_copy(&__cpu_present_mask, src);
}

void init_cpu_possible(const struct cpumask *src)
{
        cpumask_copy(&__cpu_possible_mask, src);
        __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
}

void set_cpu_online(unsigned int cpu, bool online)
{
        /*
         * atomic_inc/dec() is required to handle the horrid abuse of this
         * function by the reboot and kexec code which invoke it from
         * IPI/NMI broadcasts when shutting down CPUs. Invocation from
         * regular CPU hotplug is properly serialized.
         *
         * Note, that the fact that __num_online_cpus is of type atomic_t
         * does not protect readers which are not serialized against
         * concurrent hotplug operations.
         */
        if (online) {
                if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
                        atomic_inc(&__num_online_cpus);
        } else {
                if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
                        atomic_dec(&__num_online_cpus);
        }
}

/*
 * This should be marked __init, but there is a boatload of call sites
 * which need to be fixed up to do so. Sigh...
 */
void set_cpu_possible(unsigned int cpu, bool possible)
{
        if (possible) {
                if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
                        __num_possible_cpus++;
        } else {
                if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
                        __num_possible_cpus--;
        }
}

/*
 * Activate the first processor.
 */
void __init boot_cpu_init(void)
{
        int cpu = smp_processor_id();

        /* Mark the boot cpu "present", "online" etc for SMP and UP case */
        set_cpu_online(cpu, true);
        set_cpu_active(cpu, true);
        set_cpu_present(cpu, true);
        set_cpu_possible(cpu, true);

#ifdef CONFIG_SMP
        __boot_cpu_id = cpu;
#endif
}

/*
 * Must be called _AFTER_ setting up the per_cpu areas
 */
void __init boot_cpu_hotplug_init(void)
{
#ifdef CONFIG_SMP
        cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
        atomic_set(this_cpu_ptr(&cpuhp_state.ap_sync_state), SYNC_STATE_ONLINE);
#endif
        this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
        this_cpu_write(cpuhp_state.target, CPUHP_ONLINE);
}

#ifdef CONFIG_CPU_MITIGATIONS
/*
 * All except the cross-thread attack vector are mitigated by default.
 * Cross-thread mitigation often requires disabling SMT which is expensive
 * so cross-thread mitigations are only partially enabled by default.
 *
 * Guest-to-Host and Guest-to-Guest vectors are only needed if KVM support is
 * present.
 */
static bool attack_vectors[NR_CPU_ATTACK_VECTORS] __ro_after_init = {
        [CPU_MITIGATE_USER_KERNEL] = true,
        [CPU_MITIGATE_USER_USER] = true,
        [CPU_MITIGATE_GUEST_HOST] = IS_ENABLED(CONFIG_KVM),
        [CPU_MITIGATE_GUEST_GUEST] = IS_ENABLED(CONFIG_KVM),
};

bool cpu_attack_vector_mitigated(enum cpu_attack_vectors v)
{
        if (v < NR_CPU_ATTACK_VECTORS)
                return attack_vectors[v];

        WARN_ONCE(1, "Invalid attack vector %d\n", v);
        return false;
}

/*
 * There are 3 global options, 'off', 'auto', 'auto,nosmt'. These may optionally
 * be combined with attack-vector disables which follow them.
 *
 * Examples:
 *   mitigations=auto,no_user_kernel,no_user_user,no_cross_thread
 *   mitigations=auto,nosmt,no_guest_host,no_guest_guest
 *
 * mitigations=off is equivalent to disabling all attack vectors.
 */
enum cpu_mitigations {
        CPU_MITIGATIONS_OFF,
        CPU_MITIGATIONS_AUTO,
        CPU_MITIGATIONS_AUTO_NOSMT,
};

enum {
        NO_USER_KERNEL,
        NO_USER_USER,
        NO_GUEST_HOST,
        NO_GUEST_GUEST,
        NO_CROSS_THREAD,
        NR_VECTOR_PARAMS,
};

enum smt_mitigations smt_mitigations __ro_after_init = SMT_MITIGATIONS_AUTO;
static enum cpu_mitigations cpu_mitigations __ro_after_init = CPU_MITIGATIONS_AUTO;

static const match_table_t global_mitigations = {
        { CPU_MITIGATIONS_AUTO_NOSMT,        "auto,nosmt"},
        { CPU_MITIGATIONS_AUTO,                "auto"},
        { CPU_MITIGATIONS_OFF,                "off"},
};

static const match_table_t vector_mitigations = {
        { NO_USER_KERNEL,        "no_user_kernel"},
        { NO_USER_USER,                "no_user_user"},
        { NO_GUEST_HOST,        "no_guest_host"},
        { NO_GUEST_GUEST,        "no_guest_guest"},
        { NO_CROSS_THREAD,        "no_cross_thread"},
        { NR_VECTOR_PARAMS,        NULL},
};

static int __init mitigations_parse_global_opt(char *arg)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(global_mitigations); i++) {
                const char *pattern = global_mitigations[i].pattern;

                if (!strncmp(arg, pattern, strlen(pattern))) {
                        cpu_mitigations = global_mitigations[i].token;
                        return strlen(pattern);
                }
        }

        return 0;
}

static int __init mitigations_parse_cmdline(char *arg)
{
        char *s, *p;
        int len;

        len = mitigations_parse_global_opt(arg);

        if (cpu_mitigations_off()) {
                memset(attack_vectors, 0, sizeof(attack_vectors));
                smt_mitigations = SMT_MITIGATIONS_OFF;
        } else if (cpu_mitigations_auto_nosmt()) {
                smt_mitigations = SMT_MITIGATIONS_ON;
        }

        p = arg + len;

        if (!*p)
                return 0;

        /* Attack vector controls may come after the ',' */
        if (*p++ != ',' || !IS_ENABLED(CONFIG_ARCH_HAS_CPU_ATTACK_VECTORS)) {
                pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",        arg);
                return 0;
        }

        while ((s = strsep(&p, ",")) != NULL) {
                switch (match_token(s, vector_mitigations, NULL)) {
                case NO_USER_KERNEL:
                        attack_vectors[CPU_MITIGATE_USER_KERNEL] = false;
                        break;
                case NO_USER_USER:
                        attack_vectors[CPU_MITIGATE_USER_USER] = false;
                        break;
                case NO_GUEST_HOST:
                        attack_vectors[CPU_MITIGATE_GUEST_HOST] = false;
                        break;
                case NO_GUEST_GUEST:
                        attack_vectors[CPU_MITIGATE_GUEST_GUEST] = false;
                        break;
                case NO_CROSS_THREAD:
                        smt_mitigations = SMT_MITIGATIONS_OFF;
                        break;
                default:
                        pr_crit("Unsupported mitigations options %s\n",        s);
                        return 0;
                }
        }

        return 0;
}

/* mitigations=off */
bool cpu_mitigations_off(void)
{
        return cpu_mitigations == CPU_MITIGATIONS_OFF;
}
EXPORT_SYMBOL_GPL(cpu_mitigations_off);

/* mitigations=auto,nosmt */
bool cpu_mitigations_auto_nosmt(void)
{
        return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
}
EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);
#else
static int __init mitigations_parse_cmdline(char *arg)
{
        pr_crit("Kernel compiled without mitigations, ignoring 'mitigations'; system may still be vulnerable\n");
        return 0;
}
#endif
early_param("mitigations", mitigations_parse_cmdline);












































































    4 








    4 














    4 



















    4 






    3 
    4 







    3 
    1 



    1 












    3 






    3 



















    4 


    4 



    4 








    4 










    4 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
// SPDX-License-Identifier: GPL-2.0-only

#include <linux/phy.h>
#include <linux/phylib_stubs.h>

#include "common.h"
#include "netlink.h"

struct linkstate_req_info {
        struct ethnl_req_info                base;
};

struct linkstate_reply_data {
        struct ethnl_reply_data                        base;
        int                                        link;
        int                                        sqi;
        int                                        sqi_max;
        struct ethtool_link_ext_stats                link_stats;
        bool                                        link_ext_state_provided;
        struct ethtool_link_ext_state_info        ethtool_link_ext_state_info;
};

#define LINKSTATE_REPDATA(__reply_base) \
        container_of(__reply_base, struct linkstate_reply_data, base)

const struct nla_policy ethnl_linkstate_get_policy[] = {
        [ETHTOOL_A_LINKSTATE_HEADER]                =
                NLA_POLICY_NESTED(ethnl_header_policy_stats),
};

static int linkstate_get_sqi(struct phy_device *phydev)
{
        int ret;

        if (!phydev)
                return -EOPNOTSUPP;

        mutex_lock(&phydev->lock);
        if (!phydev->drv || !phydev->drv->get_sqi)
                ret = -EOPNOTSUPP;
        else if (!phydev->link)
                ret = -ENETDOWN;
        else
                ret = phydev->drv->get_sqi(phydev);
        mutex_unlock(&phydev->lock);

        return ret;
}

static int linkstate_get_sqi_max(struct phy_device *phydev)
{
        int ret;

        if (!phydev)
                return -EOPNOTSUPP;

        mutex_lock(&phydev->lock);
        if (!phydev->drv || !phydev->drv->get_sqi_max)
                ret = -EOPNOTSUPP;
        else if (!phydev->link)
                ret = -ENETDOWN;
        else
                ret = phydev->drv->get_sqi_max(phydev);
        mutex_unlock(&phydev->lock);

        return ret;
};

static bool linkstate_sqi_critical_error(int sqi)
{
        return sqi < 0 && sqi != -EOPNOTSUPP && sqi != -ENETDOWN;
}

static bool linkstate_sqi_valid(struct linkstate_reply_data *data)
{
        return data->sqi >= 0 && data->sqi_max >= 0 &&
               data->sqi <= data->sqi_max;
}

static int linkstate_get_link_ext_state(struct net_device *dev,
                                        struct linkstate_reply_data *data)
{
        int err;

        if (!dev->ethtool_ops->get_link_ext_state)
                return -EOPNOTSUPP;

        err = dev->ethtool_ops->get_link_ext_state(dev, &data->ethtool_link_ext_state_info);
        if (err)
                return err;

        data->link_ext_state_provided = true;

        return 0;
}

static int linkstate_prepare_data(const struct ethnl_req_info *req_base,
                                  struct ethnl_reply_data *reply_base,
                                  const struct genl_info *info)
{
        struct linkstate_reply_data *data = LINKSTATE_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        struct nlattr **tb = info->attrs;
        struct phy_device *phydev;
        int ret;

        phydev = ethnl_req_get_phydev(req_base, tb, ETHTOOL_A_LINKSTATE_HEADER,
                                      info->extack);
        if (IS_ERR(phydev))
                return PTR_ERR(phydev);

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                return ret;
        data->link = __ethtool_get_link(dev);

        ret = linkstate_get_sqi(phydev);
        if (linkstate_sqi_critical_error(ret))
                goto out;
        data->sqi = ret;

        ret = linkstate_get_sqi_max(phydev);
        if (linkstate_sqi_critical_error(ret))
                goto out;
        data->sqi_max = ret;

        if (dev->flags & IFF_UP) {
                ret = linkstate_get_link_ext_state(dev, data);
                if (ret < 0 && ret != -EOPNOTSUPP && ret != -ENODATA)
                        goto out;
        }

        ethtool_stats_init((u64 *)&data->link_stats,
                           sizeof(data->link_stats) / 8);

        if (req_base->flags & ETHTOOL_FLAG_STATS) {
                if (phydev)
                        phy_ethtool_get_link_ext_stats(phydev,
                                                       &data->link_stats);

                if (dev->ethtool_ops->get_link_ext_stats)
                        dev->ethtool_ops->get_link_ext_stats(dev,
                                                             &data->link_stats);
        }

        ret = 0;
out:
        ethnl_ops_complete(dev);
        return ret;
}

static int linkstate_reply_size(const struct ethnl_req_info *req_base,
                                const struct ethnl_reply_data *reply_base)
{
        struct linkstate_reply_data *data = LINKSTATE_REPDATA(reply_base);
        int len;

        len = nla_total_size(sizeof(u8)) /* LINKSTATE_LINK */
                + 0;

        if (linkstate_sqi_valid(data)) {
                len += nla_total_size(sizeof(u32)); /* LINKSTATE_SQI */
                len += nla_total_size(sizeof(u32)); /* LINKSTATE_SQI_MAX */
        }

        if (data->link_ext_state_provided)
                len += nla_total_size(sizeof(u8)); /* LINKSTATE_EXT_STATE */

        if (data->ethtool_link_ext_state_info.__link_ext_substate)
                len += nla_total_size(sizeof(u8)); /* LINKSTATE_EXT_SUBSTATE */

        if (data->link_stats.link_down_events != ETHTOOL_STAT_NOT_SET)
                len += nla_total_size(sizeof(u32));

        return len;
}

static int linkstate_fill_reply(struct sk_buff *skb,
                                const struct ethnl_req_info *req_base,
                                const struct ethnl_reply_data *reply_base)
{
        struct linkstate_reply_data *data = LINKSTATE_REPDATA(reply_base);

        if (data->link >= 0 &&
            nla_put_u8(skb, ETHTOOL_A_LINKSTATE_LINK, !!data->link))
                return -EMSGSIZE;

        if (linkstate_sqi_valid(data)) {
                if (nla_put_u32(skb, ETHTOOL_A_LINKSTATE_SQI, data->sqi))
                        return -EMSGSIZE;

                if (nla_put_u32(skb, ETHTOOL_A_LINKSTATE_SQI_MAX,
                                data->sqi_max))
                        return -EMSGSIZE;
        }

        if (data->link_ext_state_provided) {
                if (nla_put_u8(skb, ETHTOOL_A_LINKSTATE_EXT_STATE,
                               data->ethtool_link_ext_state_info.link_ext_state))
                        return -EMSGSIZE;

                if (data->ethtool_link_ext_state_info.__link_ext_substate &&
                    nla_put_u8(skb, ETHTOOL_A_LINKSTATE_EXT_SUBSTATE,
                               data->ethtool_link_ext_state_info.__link_ext_substate))
                        return -EMSGSIZE;
        }

        if (data->link_stats.link_down_events != ETHTOOL_STAT_NOT_SET)
                if (nla_put_u32(skb, ETHTOOL_A_LINKSTATE_EXT_DOWN_CNT,
                                data->link_stats.link_down_events))
                        return -EMSGSIZE;

        return 0;
}

const struct ethnl_request_ops ethnl_linkstate_request_ops = {
        .request_cmd                = ETHTOOL_MSG_LINKSTATE_GET,
        .reply_cmd                = ETHTOOL_MSG_LINKSTATE_GET_REPLY,
        .hdr_attr                = ETHTOOL_A_LINKSTATE_HEADER,
        .req_info_size                = sizeof(struct linkstate_req_info),
        .reply_data_size        = sizeof(struct linkstate_reply_data),

        .prepare_data                = linkstate_prepare_data,
        .reply_size                = linkstate_reply_size,
        .fill_reply                = linkstate_fill_reply,
};















































  602 







  602 










  573 
  126 











    3 



  600 
    3 












   64 






   64 


   64 










  744 










  297 
  284 
  284 






  297 

  297 





  711 



  711 








  185 
  606 









  711 









   12 






    2 
    8 



   11 





   11 















  559 













  559 






  142 










  528 


















   37 








   37 




















    8 



    8 

























  660 



  660 
  633 


   56 




  660 






    7 






    7 
    4 
    5 
    5 










    7 











   18 








   12 

    6 




   18 
   10 




   18 


   18 







  568 







  569 


  567 
  197 















































  602 


   69 
  568 
   69 




   28 


   40 











  262 




  257 


   27 


    7 


  255 





  570 





  195 














  602 


















   28 




    5 


   24 











  637 






  637 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
// SPDX-License-Identifier: GPL-2.0-or-later
/* SCTP kernel implementation
 * (C) Copyright IBM Corp. 2001, 2003
 * Copyright (c) Cisco 1999,2000
 * Copyright (c) Motorola 1999,2000,2001
 * Copyright (c) La Monte H.P. Yarroll 2001
 *
 * This file is part of the SCTP kernel implementation.
 *
 * A collection class to handle the storage of transport addresses.
 *
 * Please send any bug reports or fixes you make to the
 * email address(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *    La Monte H.P. Yarroll <piggy@acm.org>
 *    Karl Knutson          <karl@athena.chicago.il.us>
 *    Jon Grimm             <jgrimm@us.ibm.com>
 *    Daisy Chang           <daisyc@us.ibm.com>
 */

#include <linux/types.h>
#include <linux/slab.h>
#include <linux/in.h>
#include <net/sock.h>
#include <net/ipv6.h>
#include <net/if_inet6.h>
#include <net/sctp/sctp.h>
#include <net/sctp/sm.h>

/* Forward declarations for internal helpers. */
static int sctp_copy_one_addr(struct net *net, struct sctp_bind_addr *dest,
                              union sctp_addr *addr, enum sctp_scope scope,
                              gfp_t gfp, int flags);
static void sctp_bind_addr_clean(struct sctp_bind_addr *);

/* First Level Abstractions. */

/* Copy 'src' to 'dest' taking 'scope' into account.  Omit addresses
 * in 'src' which have a broader scope than 'scope'.
 */
int sctp_bind_addr_copy(struct net *net, struct sctp_bind_addr *dest,
                        const struct sctp_bind_addr *src,
                        enum sctp_scope scope, gfp_t gfp,
                        int flags)
{
        struct sctp_sockaddr_entry *addr;
        int error = 0;

        /* All addresses share the same port.  */
        dest->port = src->port;

        /* Extract the addresses which are relevant for this scope.  */
        list_for_each_entry(addr, &src->address_list, list) {
                error = sctp_copy_one_addr(net, dest, &addr->a, scope,
                                           gfp, flags);
                if (error < 0)
                        goto out;
        }

        /* If there are no addresses matching the scope and
         * this is global scope, try to get a link scope address, with
         * the assumption that we must be sitting behind a NAT.
         */
        if (list_empty(&dest->address_list) && (SCTP_SCOPE_GLOBAL == scope)) {
                list_for_each_entry(addr, &src->address_list, list) {
                        error = sctp_copy_one_addr(net, dest, &addr->a,
                                                   SCTP_SCOPE_LINK, gfp,
                                                   flags);
                        if (error < 0)
                                goto out;
                }
        }

        /* If somehow no addresses were found that can be used with this
         * scope, it's an error.
         */
        if (list_empty(&dest->address_list))
                error = -ENETUNREACH;

out:
        if (error)
                sctp_bind_addr_clean(dest);

        return error;
}

/* Exactly duplicate the address lists.  This is necessary when doing
 * peer-offs and accepts.  We don't want to put all the current system
 * addresses into the endpoint.  That's useless.  But we do want duplicat
 * the list of bound addresses that the older endpoint used.
 */
int sctp_bind_addr_dup(struct sctp_bind_addr *dest,
                        const struct sctp_bind_addr *src,
                        gfp_t gfp)
{
        struct sctp_sockaddr_entry *addr;
        int error = 0;

        /* All addresses share the same port.  */
        dest->port = src->port;

        list_for_each_entry(addr, &src->address_list, list) {
                error = sctp_add_bind_addr(dest, &addr->a, sizeof(addr->a),
                                           1, gfp);
                if (error < 0)
                        break;
        }

        return error;
}

/* Initialize the SCTP_bind_addr structure for either an endpoint or
 * an association.
 */
void sctp_bind_addr_init(struct sctp_bind_addr *bp, __u16 port)
{
        INIT_LIST_HEAD(&bp->address_list);
        bp->port = port;
}

/* Dispose of the address list. */
static void sctp_bind_addr_clean(struct sctp_bind_addr *bp)
{
        struct sctp_sockaddr_entry *addr, *temp;

        /* Empty the bind address list. */
        list_for_each_entry_safe(addr, temp, &bp->address_list, list) {
                list_del_rcu(&addr->list);
                kfree_rcu(addr, rcu);
                SCTP_DBG_OBJCNT_DEC(addr);
        }
}

/* Dispose of an SCTP_bind_addr structure  */
void sctp_bind_addr_free(struct sctp_bind_addr *bp)
{
        /* Empty the bind address list. */
        sctp_bind_addr_clean(bp);
}

/* Add an address to the bind address list in the SCTP_bind_addr structure. */
int sctp_add_bind_addr(struct sctp_bind_addr *bp, union sctp_addr *new,
                       int new_size, __u8 addr_state, gfp_t gfp)
{
        struct sctp_sockaddr_entry *addr;

        /* Add the address to the bind address list.  */
        addr = kzalloc_obj(*addr, gfp);
        if (!addr)
                return -ENOMEM;

        memcpy(&addr->a, new, min_t(size_t, sizeof(*new), new_size));

        /* Fix up the port if it has not yet been set.
         * Both v4 and v6 have the port at the same offset.
         */
        if (!addr->a.v4.sin_port)
                addr->a.v4.sin_port = htons(bp->port);

        addr->state = addr_state;
        addr->valid = 1;

        INIT_LIST_HEAD(&addr->list);

        /* We always hold a socket lock when calling this function,
         * and that acts as a writer synchronizing lock.
         */
        list_add_tail_rcu(&addr->list, &bp->address_list);
        SCTP_DBG_OBJCNT_INC(addr);

        return 0;
}

/* Delete an address from the bind address list in the SCTP_bind_addr
 * structure.
 */
int sctp_del_bind_addr(struct sctp_bind_addr *bp, union sctp_addr *del_addr)
{
        struct sctp_sockaddr_entry *addr, *temp;
        int found = 0;

        /* We hold the socket lock when calling this function,
         * and that acts as a writer synchronizing lock.
         */
        list_for_each_entry_safe(addr, temp, &bp->address_list, list) {
                if (sctp_cmp_addr_exact(&addr->a, del_addr)) {
                        /* Found the exact match. */
                        found = 1;
                        addr->valid = 0;
                        list_del_rcu(&addr->list);
                        break;
                }
        }

        if (found) {
                kfree_rcu(addr, rcu);
                SCTP_DBG_OBJCNT_DEC(addr);
                return 0;
        }

        return -EINVAL;
}

/* Create a network byte-order representation of all the addresses
 * formated as SCTP parameters.
 *
 * The second argument is the return value for the length.
 */
union sctp_params sctp_bind_addrs_to_raw(const struct sctp_bind_addr *bp,
                                         int *addrs_len,
                                         gfp_t gfp)
{
        union sctp_params addrparms;
        union sctp_params retval;
        int addrparms_len;
        union sctp_addr_param rawaddr;
        int len;
        struct sctp_sockaddr_entry *addr;
        struct list_head *pos;
        struct sctp_af *af;

        addrparms_len = 0;
        len = 0;

        /* Allocate enough memory at once. */
        list_for_each(pos, &bp->address_list) {
                len += sizeof(union sctp_addr_param);
        }

        /* Don't even bother embedding an address if there
         * is only one.
         */
        if (len == sizeof(union sctp_addr_param)) {
                retval.v = NULL;
                goto end_raw;
        }

        retval.v = kmalloc(len, gfp);
        if (!retval.v)
                goto end_raw;

        addrparms = retval;

        list_for_each_entry(addr, &bp->address_list, list) {
                af = sctp_get_af_specific(addr->a.v4.sin_family);
                len = af->to_addr_param(&addr->a, &rawaddr);
                memcpy(addrparms.v, &rawaddr, len);
                addrparms.v += len;
                addrparms_len += len;
        }

end_raw:
        *addrs_len = addrparms_len;
        return retval;
}

/*
 * Create an address list out of the raw address list format (IPv4 and IPv6
 * address parameters).
 */
int sctp_raw_to_bind_addrs(struct sctp_bind_addr *bp, __u8 *raw_addr_list,
                           int addrs_len, __u16 port, gfp_t gfp)
{
        union sctp_addr_param *rawaddr;
        struct sctp_paramhdr *param;
        union sctp_addr addr;
        int retval = 0;
        int len;
        struct sctp_af *af;

        /* Convert the raw address to standard address format */
        while (addrs_len) {
                param = (struct sctp_paramhdr *)raw_addr_list;
                rawaddr = (union sctp_addr_param *)raw_addr_list;

                if (addrs_len < sizeof(*param)) {
                        retval = -EINVAL;
                        goto out_err;
                }
                len = ntohs(param->length);
                if (addrs_len < len) {
                        retval = -EINVAL;
                        goto out_err;
                }

                af = sctp_get_af_specific(param_type2af(param->type));
                if (unlikely(!af) ||
                    !af->from_addr_param(&addr, rawaddr, htons(port), 0)) {
                        retval = -EINVAL;
                        goto out_err;
                }

                if (sctp_bind_addr_state(bp, &addr) != -1)
                        goto next;
                retval = sctp_add_bind_addr(bp, &addr, sizeof(addr),
                                            SCTP_ADDR_SRC, gfp);
                if (retval)
                        /* Can't finish building the list, clean up. */
                        goto out_err;

next:
                addrs_len -= len;
                raw_addr_list += len;
        }

        return retval;

out_err:
        if (retval)
                sctp_bind_addr_clean(bp);

        return retval;
}

/********************************************************************
 * 2nd Level Abstractions
 ********************************************************************/

/* Does this contain a specified address?  Allow wildcarding. */
int sctp_bind_addr_match(struct sctp_bind_addr *bp,
                         const union sctp_addr *addr,
                         struct sctp_sock *opt)
{
        struct sctp_sockaddr_entry *laddr;
        int match = 0;

        rcu_read_lock();
        list_for_each_entry_rcu(laddr, &bp->address_list, list) {
                if (!laddr->valid)
                        continue;
                if (opt->pf->cmp_addr(&laddr->a, addr, opt)) {
                        match = 1;
                        break;
                }
        }
        rcu_read_unlock();

        return match;
}

int sctp_bind_addrs_check(struct sctp_sock *sp,
                          struct sctp_sock *sp2, int cnt2)
{
        struct sctp_bind_addr *bp2 = &sp2->ep->base.bind_addr;
        struct sctp_bind_addr *bp = &sp->ep->base.bind_addr;
        struct sctp_sockaddr_entry *laddr, *laddr2;
        bool exist = false;
        int cnt = 0;

        rcu_read_lock();
        list_for_each_entry_rcu(laddr, &bp->address_list, list) {
                list_for_each_entry_rcu(laddr2, &bp2->address_list, list) {
                        if (sp->pf->af->cmp_addr(&laddr->a, &laddr2->a) &&
                            laddr->valid && laddr2->valid) {
                                exist = true;
                                goto next;
                        }
                }
                cnt = 0;
                break;
next:
                cnt++;
        }
        rcu_read_unlock();

        return (cnt == cnt2) ? 0 : (exist ? -EEXIST : 1);
}

/* Does the address 'addr' conflict with any addresses in
 * the bp.
 */
int sctp_bind_addr_conflict(struct sctp_bind_addr *bp,
                            const union sctp_addr *addr,
                            struct sctp_sock *bp_sp,
                            struct sctp_sock *addr_sp)
{
        struct sctp_sockaddr_entry *laddr;
        int conflict = 0;
        struct sctp_sock *sp;

        /* Pick the IPv6 socket as the basis of comparison
         * since it's usually a superset of the IPv4.
         * If there is no IPv6 socket, then default to bind_addr.
         */
        if (sctp_opt2sk(bp_sp)->sk_family == AF_INET6)
                sp = bp_sp;
        else if (sctp_opt2sk(addr_sp)->sk_family == AF_INET6)
                sp = addr_sp;
        else
                sp = bp_sp;

        rcu_read_lock();
        list_for_each_entry_rcu(laddr, &bp->address_list, list) {
                if (!laddr->valid)
                        continue;

                conflict = sp->pf->cmp_addr(&laddr->a, addr, sp);
                if (conflict)
                        break;
        }
        rcu_read_unlock();

        return conflict;
}

/* Get the state of the entry in the bind_addr_list */
int sctp_bind_addr_state(const struct sctp_bind_addr *bp,
                         const union sctp_addr *addr)
{
        struct sctp_sockaddr_entry *laddr;
        struct sctp_af *af;

        af = sctp_get_af_specific(addr->sa.sa_family);
        if (unlikely(!af))
                return -1;

        list_for_each_entry_rcu(laddr, &bp->address_list, list) {
                if (!laddr->valid)
                        continue;
                if (af->cmp_addr(&laddr->a, addr))
                        return laddr->state;
        }

        return -1;
}

/* Find the first address in the bind address list that is not present in
 * the addrs packed array.
 */
union sctp_addr *sctp_find_unmatch_addr(struct sctp_bind_addr        *bp,
                                        const union sctp_addr        *addrs,
                                        int                        addrcnt,
                                        struct sctp_sock        *opt)
{
        struct sctp_sockaddr_entry        *laddr;
        union sctp_addr                        *addr;
        void                                 *addr_buf;
        struct sctp_af                        *af;
        int                                i;

        /* This is only called sctp_send_asconf_del_ip() and we hold
         * the socket lock in that code patch, so that address list
         * can't change.
         */
        list_for_each_entry(laddr, &bp->address_list, list) {
                addr_buf = (union sctp_addr *)addrs;
                for (i = 0; i < addrcnt; i++) {
                        addr = addr_buf;
                        af = sctp_get_af_specific(addr->v4.sin_family);
                        if (!af)
                                break;

                        if (opt->pf->cmp_addr(&laddr->a, addr, opt))
                                break;

                        addr_buf += af->sockaddr_len;
                }
                if (i == addrcnt)
                        return &laddr->a;
        }

        return NULL;
}

/* Copy out addresses from the global local address list. */
static int sctp_copy_one_addr(struct net *net, struct sctp_bind_addr *dest,
                              union sctp_addr *addr, enum sctp_scope scope,
                              gfp_t gfp, int flags)
{
        int error = 0;

        if (sctp_is_any(NULL, addr)) {
                error = sctp_copy_local_addr_list(net, dest, scope, gfp, flags);
        } else if (sctp_in_scope(net, addr, scope)) {
                /* Now that the address is in scope, check to see if
                 * the address type is supported by local sock as
                 * well as the remote peer.
                 */
                if ((((AF_INET == addr->sa.sa_family) &&
                      (flags & SCTP_ADDR4_ALLOWED) &&
                      (flags & SCTP_ADDR4_PEERSUPP))) ||
                    (((AF_INET6 == addr->sa.sa_family) &&
                      (flags & SCTP_ADDR6_ALLOWED) &&
                      (flags & SCTP_ADDR6_PEERSUPP))))
                        error = sctp_add_bind_addr(dest, addr, sizeof(*addr),
                                                   SCTP_ADDR_SRC, gfp);
        }

        return error;
}

/* Is this a wildcard address?  */
int sctp_is_any(struct sock *sk, const union sctp_addr *addr)
{
        unsigned short fam = 0;
        struct sctp_af *af;

        /* Try to get the right address family */
        if (addr->sa.sa_family != AF_UNSPEC)
                fam = addr->sa.sa_family;
        else if (sk)
                fam = sk->sk_family;

        af = sctp_get_af_specific(fam);
        if (!af)
                return 0;

        return af->is_any(addr);
}

/* Is 'addr' valid for 'scope'?  */
int sctp_in_scope(struct net *net, const union sctp_addr *addr,
                  enum sctp_scope scope)
{
        enum sctp_scope addr_scope = sctp_scope(addr);

        /* The unusable SCTP addresses will not be considered with
         * any defined scopes.
         */
        if (SCTP_SCOPE_UNUSABLE == addr_scope)
                return 0;
        /*
         * For INIT and INIT-ACK address list, let L be the level of
         * requested destination address, sender and receiver
         * SHOULD include all of its addresses with level greater
         * than or equal to L.
         *
         * Address scoping can be selectively controlled via sysctl
         * option
         */
        switch (net->sctp.scope_policy) {
        case SCTP_SCOPE_POLICY_DISABLE:
                return 1;
        case SCTP_SCOPE_POLICY_ENABLE:
                if (addr_scope <= scope)
                        return 1;
                break;
        case SCTP_SCOPE_POLICY_PRIVATE:
                if (addr_scope <= scope || SCTP_SCOPE_PRIVATE == addr_scope)
                        return 1;
                break;
        case SCTP_SCOPE_POLICY_LINK:
                if (addr_scope <= scope || SCTP_SCOPE_LINK == addr_scope)
                        return 1;
                break;
        default:
                break;
        }

        return 0;
}

int sctp_is_ep_boundall(struct sock *sk)
{
        struct sctp_bind_addr *bp;
        struct sctp_sockaddr_entry *addr;

        bp = &sctp_sk(sk)->ep->base.bind_addr;
        if (sctp_list_single_entry(&bp->address_list)) {
                addr = list_entry(bp->address_list.next,
                                  struct sctp_sockaddr_entry, list);
                if (sctp_is_any(sk, &addr->a))
                        return 1;
        }
        return 0;
}

/********************************************************************
 * 3rd Level Abstractions
 ********************************************************************/

/* What is the scope of 'addr'?  */
enum sctp_scope sctp_scope(const union sctp_addr *addr)
{
        struct sctp_af *af;

        af = sctp_get_af_specific(addr->sa.sa_family);
        if (!af)
                return SCTP_SCOPE_UNUSABLE;

        return af->scope((union sctp_addr *)addr);
}













































































































































































   17 



   17 
   17 


   11 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/export.h>
#include <linux/netfilter/ipset/pfxlen.h>

/* Prefixlen maps for fast conversions, by Jan Engelhardt. */

#ifdef E
#undef E
#endif

#define PREFIXES_MAP                                                \
        E(0x00000000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0x80000000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xC0000000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xE0000000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xF0000000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xF8000000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFC000000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFE000000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFF000000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFF800000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFC00000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFE00000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFF00000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFF80000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFC0000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFE0000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFF0000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFF8000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFC000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFE000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFF000, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFF800, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFC00, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFE00, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFF00, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFF80, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFC0, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFE0, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFF0, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFF8, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFC, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFE, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0x80000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xC0000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xE0000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xF0000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xF8000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFC000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFE000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFF000000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFF800000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFC00000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFE00000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFF00000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFF80000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFC0000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFE0000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFF0000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFF8000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFC000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFE000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFF000, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFF800, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFC00, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFE00, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFF00, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFF80, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFC0, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFE0, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFF0, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFF8, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFC, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFE, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0x80000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xC0000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xE0000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xF0000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xF8000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFC000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFE000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFF000000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFF800000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFC00000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFE00000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFF00000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFF80000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFC0000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFE0000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFF0000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFF8000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFC000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFE000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFF000, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFF800, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFC00, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFE00, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFF00, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFF80, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFC0, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFE0, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF0, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF8, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFC, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFE, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x80000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xC0000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xE0000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xF0000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xF8000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFC000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFE000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFF000000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFF800000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFC00000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFE00000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFF00000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFF80000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFC0000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFE0000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFF0000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFF8000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFC000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFE000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFF000),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFF800),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFC00),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFE00),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFF00),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFF80),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFC0),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFE0),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF0),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF8),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFC),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFE),        \
        E(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF),

#define E(a, b, c, d) \
        {.ip6 = { \
                htonl(a), htonl(b), \
                htonl(c), htonl(d), \
        } }

/* This table works for both IPv4 and IPv6;
 * just use prefixlen_netmask_map[prefixlength].ip.
 */
const union nf_inet_addr ip_set_netmask_map[] = {
        PREFIXES_MAP
};
EXPORT_SYMBOL_GPL(ip_set_netmask_map);

#undef  E
#define E(a, b, c, d)                                        \
        {.ip6 = { (__force __be32)a, (__force __be32)b,        \
                  (__force __be32)c, (__force __be32)d,        \
        } }

/* This table works for both IPv4 and IPv6;
 * just use prefixlen_hostmask_map[prefixlength].ip.
 */
const union nf_inet_addr ip_set_hostmask_map[] = {
        PREFIXES_MAP
};
EXPORT_SYMBOL_GPL(ip_set_hostmask_map);

/* Find the largest network which matches the range from left, in host order. */
u32
ip_set_range_to_cidr(u32 from, u32 to, u8 *cidr)
{
        u32 last;
        u8 i;

        for (i = 1; i < 32; i++) {
                if ((from & ip_set_hostmask(i)) != from)
                        continue;
                last = from | ~ip_set_hostmask(i);
                if (!after(last, to)) {
                        *cidr = i;
                        return last;
                }
        }
        *cidr = 32;
        return from;
}
EXPORT_SYMBOL_GPL(ip_set_range_to_cidr);













  673 






















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_HEX_H
#define _LINUX_HEX_H

#include <linux/types.h>

extern const char hex_asc[];
#define hex_asc_lo(x)        hex_asc[((x) & 0x0f)]
#define hex_asc_hi(x)        hex_asc[((x) & 0xf0) >> 4]

static inline char *hex_byte_pack(char *buf, u8 byte)
{
        *buf++ = hex_asc_hi(byte);
        *buf++ = hex_asc_lo(byte);
        return buf;
}

extern const char hex_asc_upper[];
#define hex_asc_upper_lo(x)        hex_asc_upper[((x) & 0x0f)]
#define hex_asc_upper_hi(x)        hex_asc_upper[((x) & 0xf0) >> 4]

static inline char *hex_byte_pack_upper(char *buf, u8 byte)
{
        *buf++ = hex_asc_upper_hi(byte);
        *buf++ = hex_asc_upper_lo(byte);
        return buf;
}

extern int hex_to_bin(unsigned char ch);
extern int __must_check hex2bin(u8 *dst, const char *src, size_t count);
extern char *bin2hex(char *dst, const void *src, size_t count);

bool mac_pton(const char *s, u8 *mac);

#endif























































    2 
    2 



























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        cn_queue.c
 *
 * 2004+ Copyright (c) Evgeniy Polyakov <zbr@ioremap.net>
 * All rights reserved.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/skbuff.h>
#include <linux/suspend.h>
#include <linux/connector.h>
#include <linux/delay.h>

static struct cn_callback_entry *
cn_queue_alloc_callback_entry(struct cn_queue_dev *dev, const char *name,
                              const struct cb_id *id,
                              void (*callback)(struct cn_msg *,
                                               struct netlink_skb_parms *))
{
        struct cn_callback_entry *cbq;

        cbq = kzalloc_obj(*cbq);
        if (!cbq) {
                pr_err("Failed to create new callback queue.\n");
                return NULL;
        }

        refcount_set(&cbq->refcnt, 1);

        atomic_inc(&dev->refcnt);
        cbq->pdev = dev;

        snprintf(cbq->id.name, sizeof(cbq->id.name), "%s", name);
        memcpy(&cbq->id.id, id, sizeof(struct cb_id));
        cbq->callback = callback;
        return cbq;
}

void cn_queue_release_callback(struct cn_callback_entry *cbq)
{
        if (!refcount_dec_and_test(&cbq->refcnt))
                return;

        atomic_dec(&cbq->pdev->refcnt);
        kfree(cbq);
}

int cn_cb_equal(const struct cb_id *i1, const struct cb_id *i2)
{
        return ((i1->idx == i2->idx) && (i1->val == i2->val));
}

int cn_queue_add_callback(struct cn_queue_dev *dev, const char *name,
                          const struct cb_id *id,
                          void (*callback)(struct cn_msg *,
                                           struct netlink_skb_parms *))
{
        struct cn_callback_entry *cbq, *__cbq;
        int found = 0;

        cbq = cn_queue_alloc_callback_entry(dev, name, id, callback);
        if (!cbq)
                return -ENOMEM;

        spin_lock_bh(&dev->queue_lock);
        list_for_each_entry(__cbq, &dev->queue_list, callback_entry) {
                if (cn_cb_equal(&__cbq->id.id, id)) {
                        found = 1;
                        break;
                }
        }
        if (!found)
                list_add_tail(&cbq->callback_entry, &dev->queue_list);
        spin_unlock_bh(&dev->queue_lock);

        if (found) {
                cn_queue_release_callback(cbq);
                return -EINVAL;
        }

        cbq->seq = 0;
        cbq->group = cbq->id.id.idx;

        return 0;
}

void cn_queue_del_callback(struct cn_queue_dev *dev, const struct cb_id *id)
{
        struct cn_callback_entry *cbq, *n;
        int found = 0;

        spin_lock_bh(&dev->queue_lock);
        list_for_each_entry_safe(cbq, n, &dev->queue_list, callback_entry) {
                if (cn_cb_equal(&cbq->id.id, id)) {
                        list_del(&cbq->callback_entry);
                        found = 1;
                        break;
                }
        }
        spin_unlock_bh(&dev->queue_lock);

        if (found)
                cn_queue_release_callback(cbq);
}

struct cn_queue_dev *cn_queue_alloc_dev(const char *name, struct sock *nls)
{
        struct cn_queue_dev *dev;

        dev = kzalloc_obj(*dev);
        if (!dev)
                return NULL;

        snprintf(dev->name, sizeof(dev->name), "%s", name);
        atomic_set(&dev->refcnt, 0);
        INIT_LIST_HEAD(&dev->queue_list);
        spin_lock_init(&dev->queue_lock);

        dev->nls = nls;

        return dev;
}

void cn_queue_free_dev(struct cn_queue_dev *dev)
{
        struct cn_callback_entry *cbq, *n;

        spin_lock_bh(&dev->queue_lock);
        list_for_each_entry_safe(cbq, n, &dev->queue_list, callback_entry)
                list_del(&cbq->callback_entry);
        spin_unlock_bh(&dev->queue_lock);

        while (atomic_read(&dev->refcnt)) {
                pr_info("Waiting for %s to become free: refcnt=%d.\n",
                       dev->name, atomic_read(&dev->refcnt));
                msleep(1000);
        }

        kfree(dev);
        dev = NULL;
}






















































































































































































































    3 
































    1 



    1 









    1 




    1 












    1 


    1 




































































































































































































































































































    7 
























    8 
    7 
    1 












    1 



    1 






    2 
    3 

    3 
    2 
















    5 














    5 

    1 



    3 
    1 

    1 









    1 



    3 










    3 
































    7 
    7 





    7 



    5 



    3 











   10 








   16 








   16 























   25 




































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
// SPDX-License-Identifier: GPL-2.0
/* Bareudp: UDP  tunnel encasulation for different Payload types like
 * MPLS, NSH, IP, etc.
 * Copyright (c) 2019 Nokia, Inc.
 * Authors:  Martin Varghese, <martin.varghese@nokia.com>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/etherdevice.h>
#include <linux/hash.h>
#include <net/dst_metadata.h>
#include <net/gro_cells.h>
#include <net/rtnetlink.h>
#include <net/protocol.h>
#include <net/ip6_tunnel.h>
#include <net/ip_tunnels.h>
#include <net/udp_tunnel.h>
#include <net/bareudp.h>

#define BAREUDP_BASE_HLEN sizeof(struct udphdr)
#define BAREUDP_IPV4_HLEN (sizeof(struct iphdr) + \
                           sizeof(struct udphdr))
#define BAREUDP_IPV6_HLEN (sizeof(struct ipv6hdr) + \
                           sizeof(struct udphdr))

static bool log_ecn_error = true;
module_param(log_ecn_error, bool, 0644);
MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");

/* per-network namespace private data for this module */

static unsigned int bareudp_net_id;

struct bareudp_net {
        struct list_head        bareudp_list;
        struct mutex                lock;
};

struct bareudp_conf {
        __be16 ethertype;
        __be16 port;
        u16 sport_min;
        bool multi_proto_mode;
};

/* Pseudo network device */
struct bareudp_dev {
        struct net         *net;        /* netns for packet i/o */
        struct net_device  *dev;        /* netdev for bareudp tunnel */
        __be16                   ethertype;
        __be16             port;
        u16                   sport_min;
        bool               multi_proto_mode;
        struct sock        __rcu *sk;
        struct list_head   next;        /* bareudp node  on namespace list */
        struct gro_cells   gro_cells;
};

static int bareudp_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
{
        struct metadata_dst *tun_dst = NULL;
        IP_TUNNEL_DECLARE_FLAGS(key) = { };
        struct bareudp_dev *bareudp;
        unsigned short family;
        unsigned int len;
        __be16 proto;
        void *oiph;
        int err;
        int nh;

        bareudp = rcu_dereference_sk_user_data(sk);
        if (!bareudp)
                goto drop;

        if (skb->protocol ==  htons(ETH_P_IP))
                family = AF_INET;
        else
                family = AF_INET6;

        if (bareudp->ethertype == htons(ETH_P_IP)) {
                __u8 ipversion;

                if (skb_copy_bits(skb, BAREUDP_BASE_HLEN, &ipversion,
                                  sizeof(ipversion))) {
                        dev_dstats_rx_dropped(bareudp->dev);
                        goto drop;
                }
                ipversion >>= 4;

                if (ipversion == 4) {
                        proto = htons(ETH_P_IP);
                } else if (ipversion == 6 && bareudp->multi_proto_mode) {
                        proto = htons(ETH_P_IPV6);
                } else {
                        dev_dstats_rx_dropped(bareudp->dev);
                        goto drop;
                }
        } else if (bareudp->ethertype == htons(ETH_P_MPLS_UC)) {
                struct iphdr *tunnel_hdr;

                tunnel_hdr = (struct iphdr *)skb_network_header(skb);
                if (tunnel_hdr->version == 4) {
                        if (!ipv4_is_multicast(tunnel_hdr->daddr)) {
                                proto = bareudp->ethertype;
                        } else if (bareudp->multi_proto_mode &&
                                   ipv4_is_multicast(tunnel_hdr->daddr)) {
                                proto = htons(ETH_P_MPLS_MC);
                        } else {
                                dev_dstats_rx_dropped(bareudp->dev);
                                goto drop;
                        }
                } else {
                        int addr_type;
                        struct ipv6hdr *tunnel_hdr_v6;

                        tunnel_hdr_v6 = (struct ipv6hdr *)skb_network_header(skb);
                        addr_type =
                        ipv6_addr_type((struct in6_addr *)&tunnel_hdr_v6->daddr);
                        if (!(addr_type & IPV6_ADDR_MULTICAST)) {
                                proto = bareudp->ethertype;
                        } else if (bareudp->multi_proto_mode &&
                                   (addr_type & IPV6_ADDR_MULTICAST)) {
                                proto = htons(ETH_P_MPLS_MC);
                        } else {
                                dev_dstats_rx_dropped(bareudp->dev);
                                goto drop;
                        }
                }
        } else {
                proto = bareudp->ethertype;
        }

        if (iptunnel_pull_header(skb, BAREUDP_BASE_HLEN,
                                 proto,
                                 !net_eq(bareudp->net,
                                 dev_net(bareudp->dev)))) {
                dev_dstats_rx_dropped(bareudp->dev);
                goto drop;
        }

        __set_bit(IP_TUNNEL_KEY_BIT, key);

        tun_dst = udp_tun_rx_dst(skb, family, key, 0, 0);
        if (!tun_dst) {
                dev_dstats_rx_dropped(bareudp->dev);
                goto drop;
        }
        skb_dst_set(skb, &tun_dst->dst);
        skb->dev = bareudp->dev;
        skb_reset_mac_header(skb);

        /* Save offset of outer header relative to skb->head,
         * because we are going to reset the network header to the inner header
         * and might change skb->head.
         */
        nh = skb_network_header(skb) - skb->head;

        skb_reset_network_header(skb);

        if (!pskb_inet_may_pull(skb)) {
                DEV_STATS_INC(bareudp->dev, rx_length_errors);
                DEV_STATS_INC(bareudp->dev, rx_errors);
                goto drop;
        }

        /* Get the outer header. */
        oiph = skb->head + nh;

        if (!ipv6_mod_enabled() || family == AF_INET)
                err = IP_ECN_decapsulate(oiph, skb);
        else
                err = IP6_ECN_decapsulate(oiph, skb);

        if (unlikely(err)) {
                if (log_ecn_error) {
                        if  (!ipv6_mod_enabled() || family == AF_INET)
                                net_info_ratelimited("non-ECT from %pI4 "
                                                     "with TOS=%#x\n",
                                                     &((struct iphdr *)oiph)->saddr,
                                                     ((struct iphdr *)oiph)->tos);
                        else
                                net_info_ratelimited("non-ECT from %pI6\n",
                                                     &((struct ipv6hdr *)oiph)->saddr);
                }
                if (err > 1) {
                        DEV_STATS_INC(bareudp->dev, rx_frame_errors);
                        DEV_STATS_INC(bareudp->dev, rx_errors);
                        goto drop;
                }
        }

        len = skb->len;
        err = gro_cells_receive(&bareudp->gro_cells, skb);
        if (likely(err == NET_RX_SUCCESS))
                dev_dstats_rx_add(bareudp->dev, len);

        return 0;
drop:
        /* Consume bad packet */
        kfree_skb(skb);

        return 0;
}

static int bareudp_err_lookup(struct sock *sk, struct sk_buff *skb)
{
        return 0;
}

static int bareudp_init(struct net_device *dev)
{
        struct bareudp_dev *bareudp = netdev_priv(dev);
        int err;

        err = gro_cells_init(&bareudp->gro_cells, dev);
        if (err)
                return err;

        return 0;
}

static void bareudp_uninit(struct net_device *dev)
{
        struct bareudp_dev *bareudp = netdev_priv(dev);

        gro_cells_destroy(&bareudp->gro_cells);
}

static struct sock *bareudp_create_sock(struct net *net, __be16 port)
{
        struct udp_port_cfg udp_conf;
        struct socket *sock;
        int err;

        memset(&udp_conf, 0, sizeof(udp_conf));

        if (ipv6_mod_enabled())
                udp_conf.family = AF_INET6;
        else
                udp_conf.family = AF_INET;

        udp_conf.local_udp_port = port;
        /* Open UDP socket */
        err = udp_sock_create(net, &udp_conf, &sock);
        if (err < 0)
                return ERR_PTR(err);

        udp_allow_gso(sock->sk);
        return sock->sk;
}

/* Create new listen socket if needed */
static int bareudp_socket_create(struct bareudp_dev *bareudp, __be16 port)
{
        struct udp_tunnel_sock_cfg tunnel_cfg;
        struct sock *sk;

        sk = bareudp_create_sock(bareudp->net, port);
        if (IS_ERR(sk))
                return PTR_ERR(sk);

        /* Mark socket as an encapsulation socket */
        memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
        tunnel_cfg.sk_user_data = bareudp;
        tunnel_cfg.encap_type = 1;
        tunnel_cfg.encap_rcv = bareudp_udp_encap_recv;
        tunnel_cfg.encap_err_lookup = bareudp_err_lookup;
        tunnel_cfg.encap_destroy = NULL;
        setup_udp_tunnel_sock(bareudp->net, sk, &tunnel_cfg);

        rcu_assign_pointer(bareudp->sk, sk);
        return 0;
}

static int bareudp_open(struct net_device *dev)
{
        struct bareudp_dev *bareudp = netdev_priv(dev);

        return bareudp_socket_create(bareudp, bareudp->port);
}

static void bareudp_sock_release(struct bareudp_dev *bareudp)
{
        struct sock *sk;

        sk = rtnl_dereference(bareudp->sk);
        rcu_assign_pointer(bareudp->sk, NULL);
        udp_tunnel_sock_release(sk);
}

static int bareudp_stop(struct net_device *dev)
{
        struct bareudp_dev *bareudp = netdev_priv(dev);

        bareudp_sock_release(bareudp);
        return 0;
}

static int bareudp_xmit_skb(struct sk_buff *skb, struct net_device *dev,
                            struct bareudp_dev *bareudp,
                            const struct ip_tunnel_info *info)
{
        bool udp_sum = test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
        bool xnet = !net_eq(bareudp->net, dev_net(bareudp->dev));
        bool use_cache = ip_tunnel_dst_cache_usable(skb, info);
        struct sock *sk = rcu_dereference(bareudp->sk);
        const struct ip_tunnel_key *key = &info->key;
        struct rtable *rt;
        __be16 sport, df;
        int min_headroom;
        __u8 tos, ttl;
        __be32 saddr;
        int err;

        if (skb_vlan_inet_prepare(skb, skb->protocol != htons(ETH_P_TEB)))
                return -EINVAL;

        if (!sk)
                return -ESHUTDOWN;

        sport = udp_flow_src_port(bareudp->net, skb,
                                  bareudp->sport_min, USHRT_MAX,
                                  true);
        rt = udp_tunnel_dst_lookup(skb, dev, bareudp->net, 0, &saddr, &info->key,
                                   sport, bareudp->port, key->tos,
                                   use_cache ?
                                   (struct dst_cache *)&info->dst_cache : NULL);

        if (IS_ERR(rt))
                return PTR_ERR(rt);

        skb_tunnel_check_pmtu(skb, &rt->dst,
                              BAREUDP_IPV4_HLEN + info->options_len, false);

        tos = ip_tunnel_ecn_encap(key->tos, ip_hdr(skb), skb);
        ttl = key->ttl;
        df = test_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, key->tun_flags) ?
             htons(IP_DF) : 0;
        skb_scrub_packet(skb, xnet);

        err = -ENOSPC;
        if (!skb_pull(skb, skb_network_offset(skb)))
                goto free_dst;

        min_headroom = LL_RESERVED_SPACE(rt->dst.dev) + rt->dst.header_len +
                BAREUDP_BASE_HLEN + info->options_len + sizeof(struct iphdr);

        err = skb_cow_head(skb, min_headroom);
        if (unlikely(err))
                goto free_dst;

        err = udp_tunnel_handle_offloads(skb, udp_sum);
        if (err)
                goto free_dst;

        skb_set_inner_protocol(skb, bareudp->ethertype);
        udp_tunnel_xmit_skb(rt, sk, skb, saddr, info->key.u.ipv4.dst,
                            tos, ttl, df, sport, bareudp->port,
                            !net_eq(bareudp->net, dev_net(bareudp->dev)),
                            !test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags),
                            0);
        return 0;

free_dst:
        dst_release(&rt->dst);
        return err;
}

static int bareudp6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
                             struct bareudp_dev *bareudp,
                             const struct ip_tunnel_info *info)
{
        bool udp_sum = test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
        bool xnet = !net_eq(bareudp->net, dev_net(bareudp->dev));
        bool use_cache = ip_tunnel_dst_cache_usable(skb, info);
        struct sock *sk = rcu_dereference(bareudp->sk);
        const struct ip_tunnel_key *key = &info->key;
        struct dst_entry *dst = NULL;
        struct in6_addr saddr, daddr;
        int min_headroom;
        __u8 prio, ttl;
        __be16 sport;
        int err;

        if (skb_vlan_inet_prepare(skb, skb->protocol != htons(ETH_P_TEB)))
                return -EINVAL;

        if (!sk)
                return -ESHUTDOWN;

        sport = udp_flow_src_port(bareudp->net, skb,
                                  bareudp->sport_min, USHRT_MAX,
                                  true);
        dst = udp_tunnel6_dst_lookup(skb, dev, bareudp->net, sk, 0, &saddr,
                                     key, sport, bareudp->port, key->tos,
                                     use_cache ?
                                     (struct dst_cache *) &info->dst_cache : NULL);
        if (IS_ERR(dst))
                return PTR_ERR(dst);

        skb_tunnel_check_pmtu(skb, dst, BAREUDP_IPV6_HLEN + info->options_len,
                              false);

        prio = ip_tunnel_ecn_encap(key->tos, ip_hdr(skb), skb);
        ttl = key->ttl;

        skb_scrub_packet(skb, xnet);

        err = -ENOSPC;
        if (!skb_pull(skb, skb_network_offset(skb)))
                goto free_dst;

        min_headroom = LL_RESERVED_SPACE(dst->dev) + dst->header_len +
                BAREUDP_BASE_HLEN + info->options_len + sizeof(struct ipv6hdr);

        err = skb_cow_head(skb, min_headroom);
        if (unlikely(err))
                goto free_dst;

        err = udp_tunnel_handle_offloads(skb, udp_sum);
        if (err)
                goto free_dst;

        daddr = info->key.u.ipv6.dst;
        udp_tunnel6_xmit_skb(dst, sk, skb, dev,
                             &saddr, &daddr, prio, ttl,
                             info->key.label, sport, bareudp->port,
                             !test_bit(IP_TUNNEL_CSUM_BIT,
                                       info->key.tun_flags),
                             0);
        return 0;

free_dst:
        dst_release(dst);
        return err;
}

static bool bareudp_proto_valid(struct bareudp_dev *bareudp, __be16 proto)
{
        if (bareudp->ethertype == proto)
                return true;

        if (!bareudp->multi_proto_mode)
                return false;

        if (bareudp->ethertype == htons(ETH_P_MPLS_UC) &&
            proto == htons(ETH_P_MPLS_MC))
                return true;

        if (bareudp->ethertype == htons(ETH_P_IP) &&
            proto == htons(ETH_P_IPV6))
                return true;

        return false;
}

static netdev_tx_t bareudp_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct bareudp_dev *bareudp = netdev_priv(dev);
        struct ip_tunnel_info *info = NULL;
        int err;

        if (!bareudp_proto_valid(bareudp, skb->protocol)) {
                err = -EINVAL;
                goto tx_error;
        }

        info = skb_tunnel_info(skb);
        if (unlikely(!info || !(info->mode & IP_TUNNEL_INFO_TX))) {
                err = -EINVAL;
                goto tx_error;
        }

        rcu_read_lock();
        if (ipv6_mod_enabled() && info->mode & IP_TUNNEL_INFO_IPV6)
                err = bareudp6_xmit_skb(skb, dev, bareudp, info);
        else
                err = bareudp_xmit_skb(skb, dev, bareudp, info);

        rcu_read_unlock();

        if (likely(!err))
                return NETDEV_TX_OK;
tx_error:
        dev_kfree_skb(skb);

        if (err == -ELOOP)
                DEV_STATS_INC(dev, collisions);
        else if (err == -ENETUNREACH)
                DEV_STATS_INC(dev, tx_carrier_errors);

        DEV_STATS_INC(dev, tx_errors);
        return NETDEV_TX_OK;
}

static int bareudp_fill_metadata_dst(struct net_device *dev,
                                     struct sk_buff *skb)
{
        struct ip_tunnel_info *info = skb_tunnel_info(skb);
        struct bareudp_dev *bareudp = netdev_priv(dev);
        bool use_cache;
        __be16 sport;

        use_cache = ip_tunnel_dst_cache_usable(skb, info);
        sport = udp_flow_src_port(bareudp->net, skb,
                                  bareudp->sport_min, USHRT_MAX,
                                  true);

        if (!ipv6_mod_enabled() || ip_tunnel_info_af(info) == AF_INET) {
                struct rtable *rt;
                __be32 saddr;

                rt = udp_tunnel_dst_lookup(skb, dev, bareudp->net, 0, &saddr,
                                           &info->key, sport, bareudp->port,
                                           info->key.tos,
                                           use_cache ? &info->dst_cache : NULL);
                if (IS_ERR(rt))
                        return PTR_ERR(rt);

                ip_rt_put(rt);
                info->key.u.ipv4.src = saddr;
        } else if (ip_tunnel_info_af(info) == AF_INET6) {
                struct dst_entry *dst;
                struct in6_addr saddr;
                struct sock *sk;

                sk = rcu_dereference(bareudp->sk);
                if (!sk)
                        return -ESHUTDOWN;

                dst = udp_tunnel6_dst_lookup(skb, dev, bareudp->net, sk,
                                             0, &saddr, &info->key,
                                             sport, bareudp->port, info->key.tos,
                                             use_cache ? &info->dst_cache : NULL);
                if (IS_ERR(dst))
                        return PTR_ERR(dst);

                dst_release(dst);
                info->key.u.ipv6.src = saddr;
        } else {
                return -EINVAL;
        }

        info->key.tp_src = sport;
        info->key.tp_dst = bareudp->port;
        return 0;
}

static const struct net_device_ops bareudp_netdev_ops = {
        .ndo_init               = bareudp_init,
        .ndo_uninit             = bareudp_uninit,
        .ndo_open               = bareudp_open,
        .ndo_stop               = bareudp_stop,
        .ndo_start_xmit         = bareudp_xmit,
        .ndo_fill_metadata_dst  = bareudp_fill_metadata_dst,
};

static const struct nla_policy bareudp_policy[IFLA_BAREUDP_MAX + 1] = {
        [IFLA_BAREUDP_PORT]                = { .type = NLA_U16 },
        [IFLA_BAREUDP_ETHERTYPE]           = { .type = NLA_U16 },
        [IFLA_BAREUDP_SRCPORT_MIN]         = { .type = NLA_U16 },
        [IFLA_BAREUDP_MULTIPROTO_MODE]     = { .type = NLA_FLAG },
};

/* Info for udev, that this is a virtual tunnel endpoint */
static const struct device_type bareudp_type = {
        .name = "bareudp",
};

/* Initialize the device structure. */
static void bareudp_setup(struct net_device *dev)
{
        dev->netdev_ops = &bareudp_netdev_ops;
        dev->needs_free_netdev = true;
        SET_NETDEV_DEVTYPE(dev, &bareudp_type);
        dev->features    |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST;
        dev->features    |= NETIF_F_RXCSUM;
        dev->features    |= NETIF_F_GSO_SOFTWARE;
        dev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST;
        dev->hw_features |= NETIF_F_RXCSUM;
        dev->hw_features |= NETIF_F_GSO_SOFTWARE;
        dev->hard_header_len = 0;
        dev->addr_len = 0;
        dev->mtu = ETH_DATA_LEN;
        dev->min_mtu = IPV4_MIN_MTU;
        dev->max_mtu = IP_MAX_MTU - BAREUDP_BASE_HLEN;
        dev->type = ARPHRD_NONE;
        netif_keep_dst(dev);
        dev->priv_flags |= IFF_NO_QUEUE;
        dev->lltx = true;
        dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST;
        dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS;
}

static int bareudp_validate(struct nlattr *tb[], struct nlattr *data[],
                            struct netlink_ext_ack *extack)
{
        if (!data) {
                NL_SET_ERR_MSG(extack,
                               "Not enough attributes provided to perform the operation");
                return -EINVAL;
        }
        return 0;
}

static int bareudp2info(struct nlattr *data[], struct bareudp_conf *conf,
                        struct netlink_ext_ack *extack)
{
        memset(conf, 0, sizeof(*conf));

        if (!data[IFLA_BAREUDP_PORT]) {
                NL_SET_ERR_MSG(extack, "port not specified");
                return -EINVAL;
        }
        if (!data[IFLA_BAREUDP_ETHERTYPE]) {
                NL_SET_ERR_MSG(extack, "ethertype not specified");
                return -EINVAL;
        }

        conf->port = nla_get_u16(data[IFLA_BAREUDP_PORT]);
        conf->ethertype = nla_get_u16(data[IFLA_BAREUDP_ETHERTYPE]);

        if (data[IFLA_BAREUDP_SRCPORT_MIN])
                conf->sport_min =  nla_get_u16(data[IFLA_BAREUDP_SRCPORT_MIN]);

        if (data[IFLA_BAREUDP_MULTIPROTO_MODE])
                conf->multi_proto_mode = true;

        return 0;
}

static struct bareudp_dev *bareudp_find_dev(struct bareudp_net *bn,
                                            const struct bareudp_conf *conf)
{
        struct bareudp_dev *bareudp, *t = NULL;

        mutex_lock(&bn->lock);

        list_for_each_entry(bareudp, &bn->bareudp_list, next) {
                if (conf->port == bareudp->port)
                        t = bareudp;
        }

        mutex_unlock(&bn->lock);

        return t;
}

static int bareudp_configure(struct net *net, struct net_device *dev,
                             struct bareudp_conf *conf,
                             struct netlink_ext_ack *extack)
{
        struct bareudp_net *bn = net_generic(net, bareudp_net_id);
        struct bareudp_dev *t, *bareudp = netdev_priv(dev);
        int err;

        bareudp->net = net;
        bareudp->dev = dev;
        t = bareudp_find_dev(bn, conf);
        if (t) {
                NL_SET_ERR_MSG(extack, "Another bareudp device using the same port already exists");
                return -EBUSY;
        }

        if (conf->multi_proto_mode &&
            (conf->ethertype != htons(ETH_P_MPLS_UC) &&
             conf->ethertype != htons(ETH_P_IP))) {
                NL_SET_ERR_MSG(extack, "Cannot set multiproto mode for this ethertype (only IPv4 and unicast MPLS are supported)");
                return -EINVAL;
        }

        bareudp->port = conf->port;
        bareudp->ethertype = conf->ethertype;
        bareudp->sport_min = conf->sport_min;
        bareudp->multi_proto_mode = conf->multi_proto_mode;

        err = register_netdevice(dev);
        if (err)
                return err;

        mutex_lock(&bn->lock);
        list_add(&bareudp->next, &bn->bareudp_list);
        mutex_unlock(&bn->lock);

        return 0;
}

static int bareudp_link_config(struct net_device *dev,
                               struct nlattr *tb[])
{
        int err;

        if (tb[IFLA_MTU]) {
                err = dev_set_mtu(dev, nla_get_u32(tb[IFLA_MTU]));
                if (err)
                        return err;
        }
        return 0;
}

static void __bareudp_dellink(struct net *net, struct net_device *dev,
                              struct list_head *head)
{
        struct bareudp_dev *bareudp = netdev_priv(dev);

        list_del_init(&bareudp->next);
        unregister_netdevice_queue_net(net, dev, head);
}

static void bareudp_dellink(struct net_device *dev, struct list_head *head)
{
        struct bareudp_dev *bareudp = netdev_priv(dev);
        struct bareudp_net *bn;

        bn = net_generic(bareudp->net, bareudp_net_id);

        mutex_lock(&bn->lock);
        if (!list_empty(&bareudp->next))
                __bareudp_dellink(dev_net(dev), dev, head);
        mutex_unlock(&bn->lock);
}

static int bareudp_newlink(struct net_device *dev,
                           struct rtnl_newlink_params *params,
                           struct netlink_ext_ack *extack)
{
        struct net *link_net = rtnl_newlink_link_net(params);
        struct nlattr **data = params->data;
        struct nlattr **tb = params->tb;
        struct bareudp_conf conf;
        int err;

        err = bareudp2info(data, &conf, extack);
        if (err)
                return err;

        err = bareudp_configure(link_net, dev, &conf, extack);
        if (err)
                return err;

        err = bareudp_link_config(dev, tb);
        if (err)
                goto err_unconfig;

        return 0;

err_unconfig:
        bareudp_dellink(dev, NULL);
        return err;
}

static size_t bareudp_get_size(const struct net_device *dev)
{
        return  nla_total_size(sizeof(__be16)) +  /* IFLA_BAREUDP_PORT */
                nla_total_size(sizeof(__be16)) +  /* IFLA_BAREUDP_ETHERTYPE */
                nla_total_size(sizeof(__u16))  +  /* IFLA_BAREUDP_SRCPORT_MIN */
                nla_total_size(0)              +  /* IFLA_BAREUDP_MULTIPROTO_MODE */
                0;
}

static int bareudp_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
        struct bareudp_dev *bareudp = netdev_priv(dev);

        if (nla_put_be16(skb, IFLA_BAREUDP_PORT, bareudp->port))
                goto nla_put_failure;
        if (nla_put_be16(skb, IFLA_BAREUDP_ETHERTYPE, bareudp->ethertype))
                goto nla_put_failure;
        if (nla_put_u16(skb, IFLA_BAREUDP_SRCPORT_MIN, bareudp->sport_min))
                goto nla_put_failure;
        if (bareudp->multi_proto_mode &&
            nla_put_flag(skb, IFLA_BAREUDP_MULTIPROTO_MODE))
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static struct rtnl_link_ops bareudp_link_ops __read_mostly = {
        .kind           = "bareudp",
        .maxtype        = IFLA_BAREUDP_MAX,
        .policy         = bareudp_policy,
        .priv_size      = sizeof(struct bareudp_dev),
        .setup          = bareudp_setup,
        .validate       = bareudp_validate,
        .newlink        = bareudp_newlink,
        .dellink        = bareudp_dellink,
        .get_size       = bareudp_get_size,
        .fill_info      = bareudp_fill_info,
};

static __net_init int bareudp_init_net(struct net *net)
{
        struct bareudp_net *bn = net_generic(net, bareudp_net_id);

        INIT_LIST_HEAD(&bn->bareudp_list);
        mutex_init(&bn->lock);

        return 0;
}

static void __net_exit bareudp_exit_rtnl_net(struct net *net,
                                             struct list_head *dev_kill_list)
{
        struct bareudp_net *bn = net_generic(net, bareudp_net_id);
        struct bareudp_dev *bareudp, *next;

        mutex_lock(&bn->lock);

        list_for_each_entry_safe(bareudp, next, &bn->bareudp_list, next)
                __bareudp_dellink(net, bareudp->dev, dev_kill_list);

        mutex_unlock(&bn->lock);
}

static void __net_exit bareudp_exit_net(struct net *net)
{
        struct bareudp_net *bn = net_generic(net, bareudp_net_id);

        WARN_ON_ONCE(!list_empty(&bn->bareudp_list));
}

static struct pernet_operations bareudp_net_ops = {
        .init = bareudp_init_net,
        .exit_rtnl = bareudp_exit_rtnl_net,
        .exit = bareudp_exit_net,
        .id   = &bareudp_net_id,
        .size = sizeof(struct bareudp_net),
};

static int __init bareudp_init_module(void)
{
        int rc;

        rc = register_pernet_subsys(&bareudp_net_ops);
        if (rc)
                goto out1;

        rc = rtnl_link_register(&bareudp_link_ops);
        if (rc)
                goto out2;

        return 0;
out2:
        unregister_pernet_subsys(&bareudp_net_ops);
out1:
        return rc;
}
late_initcall(bareudp_init_module);

static void __exit bareudp_cleanup_module(void)
{
        rtnl_link_unregister(&bareudp_link_ops);
        unregister_pernet_subsys(&bareudp_net_ops);
}
module_exit(bareudp_cleanup_module);

MODULE_ALIAS_RTNL_LINK("bareudp");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Martin Varghese <martin.varghese@nokia.com>");
MODULE_DESCRIPTION("Interface driver for UDP encapsulated traffic");









































   81 
    4 














   75 

    5 



    4 

    1 




   81 




   81 












   79 























   81 



   81 
   81 

   79 
   79 











   21 



   21 
   21 










   34 

















   28 




   28 
   28 
    9 




   26 





    1 



    1 



   18 





    5 


    5 






    7 
    7 



    7 
    2 

    7 

    5 



    7 

    7 
    2 





   24 



    1 






   23 




















    3 















    3 








    2 




    2 








    2 















    3 



    3 



    8 









    7 





    4 




    1 



    1 





    1 



    1 


    1 






    1 






    1 
    1 






   19 


    2 

   17 














































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/sch_prio.c        Simple 3-band priority "scheduler".
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 * Fixes:       19990609: J Hadi Salim <hadi@nortelnetworks.com>:
 *              Init --  EINVAL when opt undefined
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>

struct prio_sched_data {
        int bands;
        struct tcf_proto __rcu *filter_list;
        struct tcf_block *block;
        u8  prio2band[TC_PRIO_MAX+1];
        struct Qdisc *queues[TCQ_PRIO_BANDS];
};


static struct Qdisc *
prio_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        u32 band = skb->priority;
        struct tcf_result res;
        struct tcf_proto *fl;
        int err;

        *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
        if (TC_H_MAJ(skb->priority) != sch->handle) {
                fl = rcu_dereference_bh(q->filter_list);
                err = tcf_classify_qdisc(skb, fl, &res, false);
#ifdef CONFIG_NET_CLS_ACT
                switch (err) {
                case TC_ACT_STOLEN:
                case TC_ACT_QUEUED:
                case TC_ACT_TRAP:
                        *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
                        fallthrough;
                case TC_ACT_SHOT:
                        return NULL;
                }
#endif
                if (!fl || err < 0) {
                        if (TC_H_MAJ(band))
                                band = 0;
                        return q->queues[q->prio2band[band & TC_PRIO_MAX]];
                }
                band = res.classid;
        }
        band = TC_H_MIN(band) - 1;
        if (band >= q->bands)
                return q->queues[q->prio2band[0]];

        return q->queues[band];
}

static int
prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
{
        unsigned int len = qdisc_pkt_len(skb);
        struct Qdisc *qdisc;
        int ret;

        qdisc = prio_classify(skb, sch, &ret);
#ifdef CONFIG_NET_CLS_ACT
        if (qdisc == NULL) {

                if (ret & __NET_XMIT_BYPASS)
                        qdisc_qstats_drop(sch);
                __qdisc_drop(skb, to_free);
                return ret;
        }
#endif

        ret = qdisc_enqueue(skb, qdisc, to_free);
        if (ret == NET_XMIT_SUCCESS) {
                qstats_backlog_add(sch, len);
                qdisc_qlen_inc(sch);
                return NET_XMIT_SUCCESS;
        }
        if (net_xmit_drop_count(ret))
                qdisc_qstats_drop(sch);
        return ret;
}

static struct sk_buff *prio_peek(struct Qdisc *sch)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        int prio;

        for (prio = 0; prio < q->bands; prio++) {
                struct Qdisc *qdisc = q->queues[prio];
                struct sk_buff *skb = qdisc->ops->peek(qdisc);
                if (skb)
                        return skb;
        }
        return NULL;
}

static struct sk_buff *prio_dequeue(struct Qdisc *sch)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        int prio;

        for (prio = 0; prio < q->bands; prio++) {
                struct Qdisc *qdisc = q->queues[prio];
                struct sk_buff *skb = qdisc_dequeue_peeked(qdisc);
                if (skb) {
                        qdisc_bstats_update(sch, skb);
                        qdisc_qstats_backlog_dec(sch, skb);
                        qdisc_qlen_dec(sch);
                        return skb;
                }
        }
        return NULL;

}

static void
prio_reset(struct Qdisc *sch)
{
        int prio;
        struct prio_sched_data *q = qdisc_priv(sch);

        for (prio = 0; prio < q->bands; prio++)
                qdisc_reset(q->queues[prio]);
}

static int prio_offload(struct Qdisc *sch, struct tc_prio_qopt *qopt)
{
        struct net_device *dev = qdisc_dev(sch);
        struct tc_prio_qopt_offload opt = {
                .handle = sch->handle,
                .parent = sch->parent,
        };

        if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
                return -EOPNOTSUPP;

        if (qopt) {
                opt.command = TC_PRIO_REPLACE;
                opt.replace_params.bands = qopt->bands;
                memcpy(&opt.replace_params.priomap, qopt->priomap,
                       TC_PRIO_MAX + 1);
                opt.replace_params.qstats = &sch->qstats;
        } else {
                opt.command = TC_PRIO_DESTROY;
        }

        return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_PRIO, &opt);
}

static void
prio_destroy(struct Qdisc *sch)
{
        int prio;
        struct prio_sched_data *q = qdisc_priv(sch);

        tcf_block_put(q->block);
        prio_offload(sch, NULL);
        for (prio = 0; prio < q->bands; prio++)
                qdisc_put(q->queues[prio]);
}

static int prio_tune(struct Qdisc *sch, struct nlattr *opt,
                     struct netlink_ext_ack *extack)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        struct Qdisc *queues[TCQ_PRIO_BANDS];
        int oldbands = q->bands, i;
        struct tc_prio_qopt *qopt;

        if (nla_len(opt) < sizeof(*qopt))
                return -EINVAL;
        qopt = nla_data(opt);

        if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < TCQ_MIN_PRIO_BANDS)
                return -EINVAL;

        for (i = 0; i <= TC_PRIO_MAX; i++) {
                if (qopt->priomap[i] >= qopt->bands)
                        return -EINVAL;
        }

        /* Before commit, make sure we can allocate all new qdiscs */
        for (i = oldbands; i < qopt->bands; i++) {
                queues[i] = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
                                              TC_H_MAKE(sch->handle, i + 1),
                                              extack);
                if (!queues[i]) {
                        while (i > oldbands)
                                qdisc_put(queues[--i]);
                        return -ENOMEM;
                }
        }

        prio_offload(sch, qopt);
        sch_tree_lock(sch);
        q->bands = qopt->bands;
        memcpy(q->prio2band, qopt->priomap, TC_PRIO_MAX+1);

        for (i = q->bands; i < oldbands; i++)
                qdisc_purge_queue(q->queues[i]);

        for (i = oldbands; i < q->bands; i++) {
                q->queues[i] = queues[i];
                if (q->queues[i] != &noop_qdisc)
                        qdisc_hash_add(q->queues[i], true);
        }

        sch_tree_unlock(sch);

        for (i = q->bands; i < oldbands; i++)
                qdisc_put(q->queues[i]);
        return 0;
}

static int prio_init(struct Qdisc *sch, struct nlattr *opt,
                     struct netlink_ext_ack *extack)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        int err;

        if (!opt)
                return -EINVAL;

        err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
        if (err)
                return err;

        return prio_tune(sch, opt, extack);
}

static int prio_dump_offload(struct Qdisc *sch)
{
        struct tc_prio_qopt_offload hw_stats = {
                .command = TC_PRIO_STATS,
                .handle = sch->handle,
                .parent = sch->parent,
                {
                        .stats = {
                                .bstats = &sch->bstats,
                                .qstats = &sch->qstats,
                        },
                },
        };

        return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_PRIO, &hw_stats);
}

static int prio_dump(struct Qdisc *sch, struct sk_buff *skb)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        unsigned char *b = skb_tail_pointer(skb);
        struct tc_prio_qopt opt;
        int err;

        opt.bands = q->bands;
        memcpy(&opt.priomap, q->prio2band, TC_PRIO_MAX + 1);

        err = prio_dump_offload(sch);
        if (err)
                goto nla_put_failure;

        if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
                goto nla_put_failure;

        return skb->len;

nla_put_failure:
        nlmsg_trim(skb, b);
        return -1;
}

static int prio_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
                      struct Qdisc **old, struct netlink_ext_ack *extack)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        struct tc_prio_qopt_offload graft_offload;
        unsigned long band = arg - 1;

        if (!new) {
                new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
                                        TC_H_MAKE(sch->handle, arg), extack);
                if (!new)
                        new = &noop_qdisc;
                else
                        qdisc_hash_add(new, true);
        }

        *old = qdisc_replace(sch, new, &q->queues[band]);

        graft_offload.handle = sch->handle;
        graft_offload.parent = sch->parent;
        graft_offload.graft_params.band = band;
        graft_offload.graft_params.child_handle = new->handle;
        graft_offload.command = TC_PRIO_GRAFT;

        qdisc_offload_graft_helper(qdisc_dev(sch), sch, new, *old,
                                   TC_SETUP_QDISC_PRIO, &graft_offload,
                                   extack);
        return 0;
}

static struct Qdisc *
prio_leaf(struct Qdisc *sch, unsigned long arg)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        unsigned long band = arg - 1;

        return q->queues[band];
}

static unsigned long prio_find(struct Qdisc *sch, u32 classid)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        unsigned long band = TC_H_MIN(classid);

        if (band - 1 >= q->bands)
                return 0;
        return band;
}

static unsigned long prio_bind(struct Qdisc *sch, unsigned long parent, u32 classid)
{
        return prio_find(sch, classid);
}


static void prio_unbind(struct Qdisc *q, unsigned long cl)
{
}

static int prio_dump_class(struct Qdisc *sch, unsigned long cl, struct sk_buff *skb,
                           struct tcmsg *tcm)
{
        struct prio_sched_data *q = qdisc_priv(sch);

        tcm->tcm_handle |= TC_H_MIN(cl);
        tcm->tcm_info = q->queues[cl-1]->handle;
        return 0;
}

static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
                                 struct gnet_dump *d)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        struct Qdisc *cl_q;

        cl_q = q->queues[cl - 1];
        if (gnet_stats_copy_basic(d, cl_q->cpu_bstats,
                                  &cl_q->bstats, true) < 0 ||
            qdisc_qstats_copy(d, cl_q) < 0)
                return -1;

        return 0;
}

static void prio_walk(struct Qdisc *sch, struct qdisc_walker *arg)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        int prio;

        if (arg->stop)
                return;

        for (prio = 0; prio < q->bands; prio++) {
                if (!tc_qdisc_stats_dump(sch, prio + 1, arg))
                        break;
        }
}

static struct tcf_block *prio_tcf_block(struct Qdisc *sch, unsigned long cl,
                                        struct netlink_ext_ack *extack)
{
        struct prio_sched_data *q = qdisc_priv(sch);

        if (cl)
                return NULL;
        return q->block;
}

static const struct Qdisc_class_ops prio_class_ops = {
        .graft                =        prio_graft,
        .leaf                =        prio_leaf,
        .find                =        prio_find,
        .walk                =        prio_walk,
        .tcf_block        =        prio_tcf_block,
        .bind_tcf        =        prio_bind,
        .unbind_tcf        =        prio_unbind,
        .dump                =        prio_dump_class,
        .dump_stats        =        prio_dump_class_stats,
};

static struct Qdisc_ops prio_qdisc_ops __read_mostly = {
        .next                =        NULL,
        .cl_ops                =        &prio_class_ops,
        .id                =        "prio",
        .priv_size        =        sizeof(struct prio_sched_data),
        .enqueue        =        prio_enqueue,
        .dequeue        =        prio_dequeue,
        .peek                =        prio_peek,
        .init                =        prio_init,
        .reset                =        prio_reset,
        .destroy        =        prio_destroy,
        .change                =        prio_tune,
        .dump                =        prio_dump,
        .owner                =        THIS_MODULE,
};
MODULE_ALIAS_NET_SCH("prio");

static int __init prio_module_init(void)
{
        return register_qdisc(&prio_qdisc_ops);
}

static void __exit prio_module_exit(void)
{
        unregister_qdisc(&prio_qdisc_ops);
}

module_init(prio_module_init)
module_exit(prio_module_exit)

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Simple 3-band priority qdisc");






























































































































































































    8 














































    8 







    8 





























    8 







    8 






























    8 







    8 







    8 








    8 

    8 

































































































































































































































































    7 
















































    7 


















    7 

























    7 
















    7 












    7 




































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    8 
















    8 





























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
// SPDX-License-Identifier: GPL-2.0
/*
 * linux/fs/ext4/xattr.c
 *
 * Copyright (C) 2001-2003 Andreas Gruenbacher, <agruen@suse.de>
 *
 * Fix by Harrison Xing <harrison@mountainviewdata.com>.
 * Ext4 code with a lot of help from Eric Jarman <ejarman@acm.org>.
 * Extended attributes for symlinks and special files added per
 *  suggestion of Luka Renko <luka.renko@hermes.si>.
 * xattr consolidation Copyright (c) 2004 James Morris <jmorris@redhat.com>,
 *  Red Hat Inc.
 * ea-in-inode support by Alex Tomas <alex@clusterfs.com> aka bzzz
 *  and Andreas Gruenbacher <agruen@suse.de>.
 */

/*
 * Extended attributes are stored directly in inodes (on file systems with
 * inodes bigger than 128 bytes) and on additional disk blocks. The i_file_acl
 * field contains the block number if an inode uses an additional block. All
 * attributes must fit in the inode and one additional block. Blocks that
 * contain the identical set of attributes may be shared among several inodes.
 * Identical blocks are detected by keeping a cache of blocks that have
 * recently been accessed.
 *
 * The attributes in inodes and on blocks have a different header; the entries
 * are stored in the same format:
 *
 *   +------------------+
 *   | header           |
 *   | entry 1          | |
 *   | entry 2          | | growing downwards
 *   | entry 3          | v
 *   | four null bytes  |
 *   | . . .            |
 *   | value 1          | ^
 *   | value 3          | | growing upwards
 *   | value 2          | |
 *   +------------------+
 *
 * The header is followed by multiple entry descriptors. In disk blocks, the
 * entry descriptors are kept sorted. In inodes, they are unsorted. The
 * attribute values are aligned to the end of the block in no specific order.
 *
 * Locking strategy
 * ----------------
 * EXT4_I(inode)->i_file_acl is protected by EXT4_I(inode)->xattr_sem.
 * EA blocks are only changed if they are exclusive to an inode, so
 * holding xattr_sem also means that nothing but the EA block's reference
 * count can change. Multiple writers to the same block are synchronized
 * by the buffer lock.
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/mbcache.h>
#include <linux/quotaops.h>
#include <linux/iversion.h>
#include "ext4_jbd2.h"
#include "ext4.h"
#include "xattr.h"
#include "acl.h"

#ifdef EXT4_XATTR_DEBUG
# define ea_idebug(inode, fmt, ...)                                        \
        printk(KERN_DEBUG "inode %s:%llu: " fmt "\n",                        \
               inode->i_sb->s_id, inode->i_ino, ##__VA_ARGS__)
# define ea_bdebug(bh, fmt, ...)                                        \
        printk(KERN_DEBUG "block %pg:%lu: " fmt "\n",                        \
               bh->b_bdev, (unsigned long)bh->b_blocknr, ##__VA_ARGS__)
#else
# define ea_idebug(inode, fmt, ...)        no_printk(fmt, ##__VA_ARGS__)
# define ea_bdebug(bh, fmt, ...)        no_printk(fmt, ##__VA_ARGS__)
#endif

static void ext4_xattr_block_cache_insert(struct mb_cache *,
                                          struct buffer_head *);
static struct buffer_head *
ext4_xattr_block_cache_find(struct inode *, struct ext4_xattr_header *,
                            struct mb_cache_entry **);
static __le32 ext4_xattr_hash_entry(char *name, size_t name_len, __le32 *value,
                                    size_t value_count);
static __le32 ext4_xattr_hash_entry_signed(char *name, size_t name_len, __le32 *value,
                                    size_t value_count);
static void ext4_xattr_rehash(struct ext4_xattr_header *);

static const struct xattr_handler * const ext4_xattr_handler_map[] = {
        [EXT4_XATTR_INDEX_USER]                     = &ext4_xattr_user_handler,
#ifdef CONFIG_EXT4_FS_POSIX_ACL
        [EXT4_XATTR_INDEX_POSIX_ACL_ACCESS]  = &nop_posix_acl_access,
        [EXT4_XATTR_INDEX_POSIX_ACL_DEFAULT] = &nop_posix_acl_default,
#endif
        [EXT4_XATTR_INDEX_TRUSTED]             = &ext4_xattr_trusted_handler,
#ifdef CONFIG_EXT4_FS_SECURITY
        [EXT4_XATTR_INDEX_SECURITY]             = &ext4_xattr_security_handler,
#endif
        [EXT4_XATTR_INDEX_HURD]                     = &ext4_xattr_hurd_handler,
};

const struct xattr_handler * const ext4_xattr_handlers[] = {
        &ext4_xattr_user_handler,
        &ext4_xattr_trusted_handler,
#ifdef CONFIG_EXT4_FS_SECURITY
        &ext4_xattr_security_handler,
#endif
        &ext4_xattr_hurd_handler,
        NULL
};

#define EA_BLOCK_CACHE(inode)        (((struct ext4_sb_info *) \
                                inode->i_sb->s_fs_info)->s_ea_block_cache)

#define EA_INODE_CACHE(inode)        (((struct ext4_sb_info *) \
                                inode->i_sb->s_fs_info)->s_ea_inode_cache)

#ifdef CONFIG_LOCKDEP
void ext4_xattr_inode_set_class(struct inode *ea_inode)
{
        struct ext4_inode_info *ei = EXT4_I(ea_inode);

        lockdep_set_subclass(&ea_inode->i_rwsem, 1);
        (void) ei;        /* shut up clang warning if !CONFIG_LOCKDEP */
        lockdep_set_subclass(&ei->i_data_sem, I_DATA_SEM_EA);
}
#endif

static __le32 ext4_xattr_block_csum(struct inode *inode,
                                    sector_t block_nr,
                                    struct ext4_xattr_header *hdr)
{
        struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
        __u32 csum;
        __le64 dsk_block_nr = cpu_to_le64(block_nr);
        __u32 dummy_csum = 0;
        int offset = offsetof(struct ext4_xattr_header, h_checksum);

        csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)&dsk_block_nr,
                           sizeof(dsk_block_nr));
        csum = ext4_chksum(csum, (__u8 *)hdr, offset);
        csum = ext4_chksum(csum, (__u8 *)&dummy_csum, sizeof(dummy_csum));
        offset += sizeof(dummy_csum);
        csum = ext4_chksum(csum, (__u8 *)hdr + offset,
                           EXT4_BLOCK_SIZE(inode->i_sb) - offset);

        return cpu_to_le32(csum);
}

static int ext4_xattr_block_csum_verify(struct inode *inode,
                                        struct buffer_head *bh)
{
        struct ext4_xattr_header *hdr = BHDR(bh);
        int ret = 1;

        if (ext4_has_feature_metadata_csum(inode->i_sb)) {
                lock_buffer(bh);
                ret = (hdr->h_checksum == ext4_xattr_block_csum(inode,
                                                        bh->b_blocknr, hdr));
                unlock_buffer(bh);
        }
        return ret;
}

static void ext4_xattr_block_csum_set(struct inode *inode,
                                      struct buffer_head *bh)
{
        if (ext4_has_feature_metadata_csum(inode->i_sb))
                BHDR(bh)->h_checksum = ext4_xattr_block_csum(inode,
                                                bh->b_blocknr, BHDR(bh));
}

static inline const char *ext4_xattr_prefix(int name_index,
                                            struct dentry *dentry)
{
        const struct xattr_handler *handler = NULL;

        if (name_index > 0 && name_index < ARRAY_SIZE(ext4_xattr_handler_map))
                handler = ext4_xattr_handler_map[name_index];

        if (!xattr_handler_can_list(handler, dentry))
                return NULL;

        return xattr_prefix(handler);
}

static int
check_xattrs(struct inode *inode, struct buffer_head *bh,
             struct ext4_xattr_entry *entry, void *end, void *value_start,
             const char *function, unsigned int line)
{
        struct ext4_xattr_entry *e = entry;
        int err = -EFSCORRUPTED;
        char *err_str;

        if (bh) {
                if (BHDR(bh)->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC) ||
                    BHDR(bh)->h_blocks != cpu_to_le32(1)) {
                        err_str = "invalid header";
                        goto errout;
                }
                if (buffer_verified(bh))
                        return 0;
                if (!ext4_xattr_block_csum_verify(inode, bh)) {
                        err = -EFSBADCRC;
                        err_str = "invalid checksum";
                        goto errout;
                }
        } else {
                struct ext4_xattr_ibody_header *header = value_start;

                header -= 1;
                if (end - (void *)header < sizeof(*header) + sizeof(u32)) {
                        err_str = "in-inode xattr block too small";
                        goto errout;
                }
                if (header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
                        err_str = "bad magic number in in-inode xattr";
                        goto errout;
                }
        }

        /* Find the end of the names list */
        while (!IS_LAST_ENTRY(e)) {
                struct ext4_xattr_entry *next = EXT4_XATTR_NEXT(e);
                if ((void *)next + sizeof(u32) > end) {
                        err_str = "e_name out of bounds";
                        goto errout;
                }
                if (strnlen(e->e_name, e->e_name_len) != e->e_name_len) {
                        err_str = "bad e_name length";
                        goto errout;
                }
                e = next;
        }

        /* Check the values */
        while (!IS_LAST_ENTRY(entry)) {
                u32 size = le32_to_cpu(entry->e_value_size);
                unsigned long ea_ino = le32_to_cpu(entry->e_value_inum);

                if (!ext4_has_feature_ea_inode(inode->i_sb) && ea_ino) {
                        err_str = "ea_inode specified without ea_inode feature enabled";
                        goto errout;
                }
                if (ea_ino && ((ea_ino == EXT4_ROOT_INO) ||
                               !ext4_valid_inum(inode->i_sb, ea_ino))) {
                        err_str = "invalid ea_ino";
                        goto errout;
                }
                if (ea_ino && !size) {
                        err_str = "invalid size in ea xattr";
                        goto errout;
                }
                if (size > EXT4_XATTR_SIZE_MAX) {
                        err_str = "e_value size too large";
                        goto errout;
                }

                if (size != 0 && entry->e_value_inum == 0) {
                        u16 offs = le16_to_cpu(entry->e_value_offs);
                        void *value;

                        /*
                         * The value cannot overlap the names, and the value
                         * with padding cannot extend beyond 'end'.  Check both
                         * the padded and unpadded sizes, since the size may
                         * overflow to 0 when adding padding.
                         */
                        if (offs > end - value_start) {
                                err_str = "e_value out of bounds";
                                goto errout;
                        }
                        value = value_start + offs;
                        if (value < (void *)e + sizeof(u32) ||
                            size > end - value ||
                            EXT4_XATTR_SIZE(size) > end - value) {
                                err_str = "overlapping e_value ";
                                goto errout;
                        }
                }
                entry = EXT4_XATTR_NEXT(entry);
        }
        if (bh)
                set_buffer_verified(bh);
        return 0;

errout:
        if (bh)
                __ext4_error_inode(inode, function, line, 0, -err,
                                   "corrupted xattr block %llu: %s",
                                   (unsigned long long) bh->b_blocknr,
                                   err_str);
        else
                __ext4_error_inode(inode, function, line, 0, -err,
                                   "corrupted in-inode xattr: %s", err_str);
        return err;
}

static inline int
__ext4_xattr_check_block(struct inode *inode, struct buffer_head *bh,
                         const char *function, unsigned int line)
{
        return check_xattrs(inode, bh, BFIRST(bh), bh->b_data + bh->b_size,
                            bh->b_data, function, line);
}

#define ext4_xattr_check_block(inode, bh) \
        __ext4_xattr_check_block((inode), (bh),  __func__, __LINE__)


int
__xattr_check_inode(struct inode *inode, struct ext4_xattr_ibody_header *header,
                         void *end, const char *function, unsigned int line)
{
        return check_xattrs(inode, NULL, IFIRST(header), end, IFIRST(header),
                            function, line);
}

static int
xattr_find_entry(struct inode *inode, struct ext4_xattr_entry **pentry,
                 void *end, int name_index, const char *name, int sorted)
{
        struct ext4_xattr_entry *entry, *next;
        size_t name_len;
        int cmp = 1;

        if (name == NULL)
                return -EINVAL;
        name_len = strlen(name);
        for (entry = *pentry; !IS_LAST_ENTRY(entry); entry = next) {
                next = EXT4_XATTR_NEXT(entry);
                if ((void *) next >= end) {
                        EXT4_ERROR_INODE(inode, "corrupted xattr entries");
                        return -EFSCORRUPTED;
                }
                cmp = name_index - entry->e_name_index;
                if (!cmp)
                        cmp = name_len - entry->e_name_len;
                if (!cmp)
                        cmp = memcmp(name, entry->e_name, name_len);
                if (!cmp || (cmp < 0 && sorted))
                        break;
        }
        *pentry = entry;
        return cmp ? -ENODATA : 0;
}

static u32
ext4_xattr_inode_hash(struct ext4_sb_info *sbi, const void *buffer, size_t size)
{
        return ext4_chksum(sbi->s_csum_seed, buffer, size);
}

static u64 ext4_xattr_inode_get_ref(struct inode *ea_inode)
{
        return ((u64) inode_get_ctime_sec(ea_inode) << 32) |
                (u32) inode_peek_iversion_raw(ea_inode);
}

static void ext4_xattr_inode_set_ref(struct inode *ea_inode, u64 ref_count)
{
        inode_set_ctime(ea_inode, (u32)(ref_count >> 32), 0);
        inode_set_iversion_raw(ea_inode, ref_count & 0xffffffff);
}

static u32 ext4_xattr_inode_get_hash(struct inode *ea_inode)
{
        return (u32) inode_get_atime_sec(ea_inode);
}

static void ext4_xattr_inode_set_hash(struct inode *ea_inode, u32 hash)
{
        inode_set_atime(ea_inode, hash, 0);
}

/*
 * Read the EA value from an inode.
 */
static int ext4_xattr_inode_read(struct inode *ea_inode, void *buf, size_t size)
{
        int blocksize = 1 << ea_inode->i_blkbits;
        int bh_count = (size + blocksize - 1) >> ea_inode->i_blkbits;
        int tail_size = (size % blocksize) ?: blocksize;
        struct buffer_head *bhs_inline[8];
        struct buffer_head **bhs = bhs_inline;
        int i, ret;

        if (bh_count > ARRAY_SIZE(bhs_inline)) {
                bhs = kmalloc_objs(*bhs, bh_count, GFP_NOFS);
                if (!bhs)
                        return -ENOMEM;
        }

        ret = ext4_bread_batch(ea_inode, 0 /* block */, bh_count,
                               true /* wait */, bhs);
        if (ret)
                goto free_bhs;

        for (i = 0; i < bh_count; i++) {
                /* There shouldn't be any holes in ea_inode. */
                if (!bhs[i]) {
                        ret = -EFSCORRUPTED;
                        goto put_bhs;
                }
                memcpy((char *)buf + blocksize * i, bhs[i]->b_data,
                       i < bh_count - 1 ? blocksize : tail_size);
        }
        ret = 0;
put_bhs:
        for (i = 0; i < bh_count; i++)
                brelse(bhs[i]);
free_bhs:
        if (bhs != bhs_inline)
                kfree(bhs);
        return ret;
}

#define EXT4_XATTR_INODE_GET_PARENT(inode) ((__u32)(inode_get_mtime_sec(inode)))

static int ext4_xattr_inode_iget(struct inode *parent, unsigned long ea_ino,
                                 u32 ea_inode_hash, struct inode **ea_inode)
{
        struct inode *inode;
        int err;

        /*
         * We have to check for this corruption early as otherwise
         * iget_locked() could wait indefinitely for the state of our
         * parent inode.
         */
        if (parent->i_ino == ea_ino) {
                ext4_error(parent->i_sb,
                           "Parent and EA inode have the same ino %lu", ea_ino);
                return -EFSCORRUPTED;
        }

        inode = ext4_iget(parent->i_sb, ea_ino, EXT4_IGET_EA_INODE);
        if (IS_ERR(inode)) {
                err = PTR_ERR(inode);
                ext4_error(parent->i_sb,
                           "error while reading EA inode %lu err=%d", ea_ino,
                           err);
                return err;
        }
        ext4_xattr_inode_set_class(inode);

        /*
         * Check whether this is an old Lustre-style xattr inode. Lustre
         * implementation does not have hash validation, rather it has a
         * backpointer from ea_inode to the parent inode.
         */
        if (ea_inode_hash != ext4_xattr_inode_get_hash(inode) &&
            EXT4_XATTR_INODE_GET_PARENT(inode) == parent->i_ino &&
            inode->i_generation == parent->i_generation) {
                ext4_set_inode_state(inode, EXT4_STATE_LUSTRE_EA_INODE);
                ext4_xattr_inode_set_ref(inode, 1);
        } else {
                inode_lock_nested(inode, I_MUTEX_XATTR);
                inode->i_flags |= S_NOQUOTA;
                inode_unlock(inode);
        }

        /*
         * Since this function resolves references from active xattr entries,
         * the EA inode must be in active state (i_nlink=1, ref_count>0).
         * i_nlink > 1, i_nlink == 0 (dangling reference), or ref_count == 0
         * (inconsistent with an active entry) all indicate on-disk corruption.
         */
        if (inode->i_nlink != 1 || !ext4_xattr_inode_get_ref(inode)) {
                ext4_error(parent->i_sb,
                           "EA inode %lu has unexpected i_nlink=%u ref_count=%llu",
                           ea_ino, inode->i_nlink,
                           ext4_xattr_inode_get_ref(inode));
                ext4_put_ea_inode(inode);
                return -EFSCORRUPTED;
        }

        *ea_inode = inode;
        return 0;
}

/* Remove entry from mbcache when EA inode is getting evicted */
void ext4_evict_ea_inode(struct inode *inode)
{
        struct mb_cache_entry *oe;

        if (!EA_INODE_CACHE(inode))
                return;
        /* Wait for entry to get unused so that we can remove it */
        while ((oe = mb_cache_entry_delete_or_get(EA_INODE_CACHE(inode),
                        ext4_xattr_inode_get_hash(inode), inode->i_ino))) {
                mb_cache_entry_wait_unused(oe);
                mb_cache_entry_put(EA_INODE_CACHE(inode), oe);
        }
}

static int
ext4_xattr_inode_verify_hashes(struct inode *ea_inode,
                               struct ext4_xattr_entry *entry, void *buffer,
                               size_t size)
{
        u32 hash;

        /* Verify stored hash matches calculated hash. */
        hash = ext4_xattr_inode_hash(EXT4_SB(ea_inode->i_sb), buffer, size);
        if (hash != ext4_xattr_inode_get_hash(ea_inode))
                return -EFSCORRUPTED;

        if (entry) {
                __le32 e_hash, tmp_data;

                /* Verify entry hash. */
                tmp_data = cpu_to_le32(hash);
                e_hash = ext4_xattr_hash_entry(entry->e_name, entry->e_name_len,
                                               &tmp_data, 1);
                /* All good? */
                if (e_hash == entry->e_hash)
                        return 0;

                /*
                 * Not good. Maybe the entry hash was calculated
                 * using the buggy signed char version?
                 */
                e_hash = ext4_xattr_hash_entry_signed(entry->e_name, entry->e_name_len,
                                                        &tmp_data, 1);
                /* Still no match - bad */
                if (e_hash != entry->e_hash)
                        return -EFSCORRUPTED;

                /* Let people know about old hash */
                pr_warn_once("ext4: filesystem with signed xattr name hash");
        }
        return 0;
}

/*
 * Read xattr value from the EA inode.
 */
static int
ext4_xattr_inode_get(struct inode *inode, struct ext4_xattr_entry *entry,
                     void *buffer, size_t size)
{
        struct mb_cache *ea_inode_cache = EA_INODE_CACHE(inode);
        struct inode *ea_inode;
        int err;

        err = ext4_xattr_inode_iget(inode, le32_to_cpu(entry->e_value_inum),
                                    le32_to_cpu(entry->e_hash), &ea_inode);
        if (err) {
                ea_inode = NULL;
                goto out;
        }

        if (i_size_read(ea_inode) != size) {
                ext4_warning_inode(ea_inode,
                                   "ea_inode file size=%llu entry size=%zu",
                                   i_size_read(ea_inode), size);
                err = -EFSCORRUPTED;
                goto out;
        }

        err = ext4_xattr_inode_read(ea_inode, buffer, size);
        if (err)
                goto out;

        if (!ext4_test_inode_state(ea_inode, EXT4_STATE_LUSTRE_EA_INODE)) {
                err = ext4_xattr_inode_verify_hashes(ea_inode, entry, buffer,
                                                     size);
                if (err) {
                        ext4_warning_inode(ea_inode,
                                           "EA inode hash validation failed");
                        goto out;
                }

                if (ea_inode_cache)
                        mb_cache_entry_create(ea_inode_cache, GFP_NOFS,
                                        ext4_xattr_inode_get_hash(ea_inode),
                                        ea_inode->i_ino, true /* reusable */);
        }
out:
        ext4_put_ea_inode(ea_inode);
        return err;
}

static int
ext4_xattr_block_get(struct inode *inode, int name_index, const char *name,
                     void *buffer, size_t buffer_size)
{
        struct buffer_head *bh = NULL;
        struct ext4_xattr_entry *entry;
        size_t size;
        void *end;
        int error;
        struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode);

        ea_idebug(inode, "name=%d.%s, buffer=%p, buffer_size=%ld",
                  name_index, name, buffer, (long)buffer_size);

        if (!EXT4_I(inode)->i_file_acl)
                return -ENODATA;
        ea_idebug(inode, "reading block %llu",
                  (unsigned long long)EXT4_I(inode)->i_file_acl);
        bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO);
        if (IS_ERR(bh))
                return PTR_ERR(bh);
        ea_bdebug(bh, "b_count=%d, refcount=%d",
                atomic_read(&(bh->b_count)), le32_to_cpu(BHDR(bh)->h_refcount));
        error = ext4_xattr_check_block(inode, bh);
        if (error)
                goto cleanup;
        ext4_xattr_block_cache_insert(ea_block_cache, bh);
        entry = BFIRST(bh);
        end = bh->b_data + bh->b_size;
        error = xattr_find_entry(inode, &entry, end, name_index, name, 1);
        if (error)
                goto cleanup;
        size = le32_to_cpu(entry->e_value_size);
        error = -ERANGE;
        if (unlikely(size > EXT4_XATTR_SIZE_MAX))
                goto cleanup;
        if (buffer) {
                if (size > buffer_size)
                        goto cleanup;
                if (entry->e_value_inum) {
                        error = ext4_xattr_inode_get(inode, entry, buffer,
                                                     size);
                        if (error)
                                goto cleanup;
                } else {
                        u16 offset = le16_to_cpu(entry->e_value_offs);
                        void *p = bh->b_data + offset;

                        if (unlikely(p + size > end))
                                goto cleanup;
                        memcpy(buffer, p, size);
                }
        }
        error = size;

cleanup:
        brelse(bh);
        return error;
}

int
ext4_xattr_ibody_get(struct inode *inode, int name_index, const char *name,
                     void *buffer, size_t buffer_size)
{
        struct ext4_xattr_ibody_header *header;
        struct ext4_xattr_entry *entry;
        struct ext4_inode *raw_inode;
        struct ext4_iloc iloc;
        size_t size;
        void *end;
        int error;

        if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR))
                return -ENODATA;
        error = ext4_get_inode_loc(inode, &iloc);
        if (error)
                return error;
        raw_inode = ext4_raw_inode(&iloc);
        header = IHDR(inode, raw_inode);
        end = ITAIL(inode, raw_inode);
        entry = IFIRST(header);
        error = xattr_find_entry(inode, &entry, end, name_index, name, 0);
        if (error)
                goto cleanup;
        size = le32_to_cpu(entry->e_value_size);
        error = -ERANGE;
        if (unlikely(size > EXT4_XATTR_SIZE_MAX))
                goto cleanup;
        if (buffer) {
                if (size > buffer_size)
                        goto cleanup;
                if (entry->e_value_inum) {
                        error = ext4_xattr_inode_get(inode, entry, buffer,
                                                     size);
                        if (error)
                                goto cleanup;
                } else {
                        u16 offset = le16_to_cpu(entry->e_value_offs);
                        void *p = (void *)IFIRST(header) + offset;

                        if (unlikely(p + size > end))
                                goto cleanup;
                        memcpy(buffer, p, size);
                }
        }
        error = size;

cleanup:
        brelse(iloc.bh);
        return error;
}

/*
 * ext4_xattr_get()
 *
 * Copy an extended attribute into the buffer
 * provided, or compute the buffer size required.
 * Buffer is NULL to compute the size of the buffer required.
 *
 * Returns a negative error number on failure, or the number of bytes
 * used / required on success.
 */
int
ext4_xattr_get(struct inode *inode, int name_index, const char *name,
               void *buffer, size_t buffer_size)
{
        int error;

        if (unlikely(ext4_forced_shutdown(inode->i_sb)))
                return -EIO;

        if (strlen(name) > 255)
                return -ERANGE;

        down_read(&EXT4_I(inode)->xattr_sem);
        error = ext4_xattr_ibody_get(inode, name_index, name, buffer,
                                     buffer_size);
        if (error == -ENODATA)
                error = ext4_xattr_block_get(inode, name_index, name, buffer,
                                             buffer_size);
        up_read(&EXT4_I(inode)->xattr_sem);
        return error;
}

static int
ext4_xattr_list_entries(struct dentry *dentry, struct ext4_xattr_entry *entry,
                        char *buffer, size_t buffer_size)
{
        size_t rest = buffer_size;

        for (; !IS_LAST_ENTRY(entry); entry = EXT4_XATTR_NEXT(entry)) {
                const char *prefix;

                prefix = ext4_xattr_prefix(entry->e_name_index, dentry);
                if (prefix) {
                        size_t prefix_len = strlen(prefix);
                        size_t size = prefix_len + entry->e_name_len + 1;

                        if (buffer) {
                                if (size > rest)
                                        return -ERANGE;
                                memcpy(buffer, prefix, prefix_len);
                                buffer += prefix_len;
                                memcpy(buffer, entry->e_name, entry->e_name_len);
                                buffer += entry->e_name_len;
                                *buffer++ = 0;
                        }
                        rest -= size;
                }
        }
        return buffer_size - rest;  /* total size */
}

static int
ext4_xattr_block_list(struct dentry *dentry, char *buffer, size_t buffer_size)
{
        struct inode *inode = d_inode(dentry);
        struct buffer_head *bh = NULL;
        int error;

        ea_idebug(inode, "buffer=%p, buffer_size=%ld",
                  buffer, (long)buffer_size);

        if (!EXT4_I(inode)->i_file_acl)
                return 0;
        ea_idebug(inode, "reading block %llu",
                  (unsigned long long)EXT4_I(inode)->i_file_acl);
        bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO);
        if (IS_ERR(bh))
                return PTR_ERR(bh);
        ea_bdebug(bh, "b_count=%d, refcount=%d",
                atomic_read(&(bh->b_count)), le32_to_cpu(BHDR(bh)->h_refcount));
        error = ext4_xattr_check_block(inode, bh);
        if (error)
                goto cleanup;
        ext4_xattr_block_cache_insert(EA_BLOCK_CACHE(inode), bh);
        error = ext4_xattr_list_entries(dentry, BFIRST(bh), buffer,
                                        buffer_size);
cleanup:
        brelse(bh);
        return error;
}

static int
ext4_xattr_ibody_list(struct dentry *dentry, char *buffer, size_t buffer_size)
{
        struct inode *inode = d_inode(dentry);
        struct ext4_xattr_ibody_header *header;
        struct ext4_inode *raw_inode;
        struct ext4_iloc iloc;
        int error;

        if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR))
                return 0;
        error = ext4_get_inode_loc(inode, &iloc);
        if (error)
                return error;
        raw_inode = ext4_raw_inode(&iloc);
        header = IHDR(inode, raw_inode);
        error = ext4_xattr_list_entries(dentry, IFIRST(header),
                                        buffer, buffer_size);

        brelse(iloc.bh);
        return error;
}

/*
 * Inode operation listxattr()
 *
 * d_inode(dentry)->i_rwsem: don't care
 *
 * Copy a list of attribute names into the buffer
 * provided, or compute the buffer size required.
 * Buffer is NULL to compute the size of the buffer required.
 *
 * Returns a negative error number on failure, or the number of bytes
 * used / required on success.
 */
ssize_t
ext4_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size)
{
        int ret, ret2;

        down_read(&EXT4_I(d_inode(dentry))->xattr_sem);
        ret = ret2 = ext4_xattr_ibody_list(dentry, buffer, buffer_size);
        if (ret < 0)
                goto errout;
        if (buffer) {
                buffer += ret;
                buffer_size -= ret;
        }
        ret = ext4_xattr_block_list(dentry, buffer, buffer_size);
        if (ret < 0)
                goto errout;
        ret += ret2;
errout:
        up_read(&EXT4_I(d_inode(dentry))->xattr_sem);
        return ret;
}

/*
 * If the EXT4_FEATURE_COMPAT_EXT_ATTR feature of this file system is
 * not set, set it.
 */
static void ext4_xattr_update_super_block(handle_t *handle,
                                          struct super_block *sb)
{
        if (ext4_has_feature_xattr(sb))
                return;

        BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get_write_access");
        if (ext4_journal_get_write_access(handle, sb, EXT4_SB(sb)->s_sbh,
                                          EXT4_JTR_NONE) == 0) {
                lock_buffer(EXT4_SB(sb)->s_sbh);
                ext4_set_feature_xattr(sb);
                ext4_superblock_csum_set(sb);
                unlock_buffer(EXT4_SB(sb)->s_sbh);
                ext4_handle_dirty_metadata(handle, NULL, EXT4_SB(sb)->s_sbh);
        }
}

int ext4_get_inode_usage(struct inode *inode, qsize_t *usage)
{
        struct ext4_iloc iloc = { .bh = NULL };
        struct buffer_head *bh = NULL;
        struct ext4_inode *raw_inode;
        struct ext4_xattr_ibody_header *header;
        struct ext4_xattr_entry *entry;
        qsize_t ea_inode_refs = 0;
        int ret;

        lockdep_assert_held_read(&EXT4_I(inode)->xattr_sem);

        if (ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {
                ret = ext4_get_inode_loc(inode, &iloc);
                if (ret)
                        goto out;
                raw_inode = ext4_raw_inode(&iloc);
                header = IHDR(inode, raw_inode);

                for (entry = IFIRST(header); !IS_LAST_ENTRY(entry);
                     entry = EXT4_XATTR_NEXT(entry))
                        if (entry->e_value_inum)
                                ea_inode_refs++;
        }

        if (EXT4_I(inode)->i_file_acl) {
                bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO);
                if (IS_ERR(bh)) {
                        ret = PTR_ERR(bh);
                        bh = NULL;
                        goto out;
                }

                ret = ext4_xattr_check_block(inode, bh);
                if (ret)
                        goto out;

                for (entry = BFIRST(bh); !IS_LAST_ENTRY(entry);
                     entry = EXT4_XATTR_NEXT(entry))
                        if (entry->e_value_inum)
                                ea_inode_refs++;
        }
        *usage = ea_inode_refs + 1;
        ret = 0;
out:
        brelse(iloc.bh);
        brelse(bh);
        return ret;
}

static inline size_t round_up_cluster(struct inode *inode, size_t length)
{
        struct super_block *sb = inode->i_sb;
        size_t cluster_size = 1 << (EXT4_SB(sb)->s_cluster_bits +
                                    inode->i_blkbits);
        size_t mask = ~(cluster_size - 1);

        return (length + cluster_size - 1) & mask;
}

static int ext4_xattr_inode_alloc_quota(struct inode *inode, size_t len)
{
        int err;

        err = dquot_alloc_inode(inode);
        if (err)
                return err;
        err = dquot_alloc_space_nodirty(inode, round_up_cluster(inode, len));
        if (err)
                dquot_free_inode(inode);
        return err;
}

static void ext4_xattr_inode_free_quota(struct inode *parent,
                                        struct inode *ea_inode,
                                        size_t len)
{
        if (ea_inode &&
            ext4_test_inode_state(ea_inode, EXT4_STATE_LUSTRE_EA_INODE))
                return;
        dquot_free_space_nodirty(parent, round_up_cluster(parent, len));
        dquot_free_inode(parent);
}

int __ext4_xattr_set_credits(struct super_block *sb, struct inode *inode,
                             struct buffer_head *block_bh, size_t value_len,
                             bool is_create)
{
        int credits;
        int blocks;

        /*
         * 1) Owner inode update
         * 2) Ref count update on old xattr block
         * 3) new xattr block
         * 4) block bitmap update for new xattr block
         * 5) group descriptor for new xattr block
         * 6) block bitmap update for old xattr block
         * 7) group descriptor for old block
         *
         * 6 & 7 can happen if we have two racing threads T_a and T_b
         * which are each trying to set an xattr on inodes I_a and I_b
         * which were both initially sharing an xattr block.
         */
        credits = 7;

        /* Quota updates. */
        credits += EXT4_MAXQUOTAS_TRANS_BLOCKS(sb);

        /*
         * In case of inline data, we may push out the data to a block,
         * so we need to reserve credits for this eventuality
         */
        if (inode && ext4_has_inline_data(inode))
                credits += ext4_chunk_trans_extent(inode, 1) + 1;

        /* We are done if ea_inode feature is not enabled. */
        if (!ext4_has_feature_ea_inode(sb))
                return credits;

        /* New ea_inode, inode map, block bitmap, group descriptor. */
        credits += 4;

        /* Data blocks. */
        blocks = (value_len + sb->s_blocksize - 1) >> sb->s_blocksize_bits;

        /* Indirection block or one level of extent tree. */
        blocks += 1;

        /* Block bitmap and group descriptor updates for each block. */
        credits += blocks * 2;

        /* Blocks themselves. */
        credits += blocks;

        if (!is_create) {
                /* Dereference ea_inode holding old xattr value.
                 * Old ea_inode, inode map, block bitmap, group descriptor.
                 */
                credits += 4;

                /* Data blocks for old ea_inode. */
                blocks = XATTR_SIZE_MAX >> sb->s_blocksize_bits;

                /* Indirection block or one level of extent tree for old
                 * ea_inode.
                 */
                blocks += 1;

                /* Block bitmap and group descriptor updates for each block. */
                credits += blocks * 2;
        }

        /* We may need to clone the existing xattr block in which case we need
         * to increment ref counts for existing ea_inodes referenced by it.
         */
        if (block_bh) {
                struct ext4_xattr_entry *entry = BFIRST(block_bh);

                for (; !IS_LAST_ENTRY(entry); entry = EXT4_XATTR_NEXT(entry))
                        if (entry->e_value_inum)
                                /* Ref count update on ea_inode. */
                                credits += 1;
        }
        return credits;
}

static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode,
                                       int ref_change)
{
        struct ext4_iloc iloc;
        u64 ref_count;
        int ret;

        inode_lock_nested(ea_inode, I_MUTEX_XATTR);

        ret = ext4_reserve_inode_write(handle, ea_inode, &iloc);
        if (ret)
                goto out;

        ref_count = ext4_xattr_inode_get_ref(ea_inode);
        if ((ref_count == 0 && ref_change < 0) || (ref_count == U64_MAX && ref_change > 0)) {
                ext4_error_inode(ea_inode, __func__, __LINE__, 0,
                        "EA inode %llu ref wraparound: ref_count=%lld ref_change=%d",
                        ea_inode->i_ino, ref_count, ref_change);
                brelse(iloc.bh);
                ret = -EFSCORRUPTED;
                goto out;
        }
        ref_count += ref_change;
        ext4_xattr_inode_set_ref(ea_inode, ref_count);

        if (ref_change > 0) {
                if (ref_count == 1) {
                        WARN_ONCE(ea_inode->i_nlink, "EA inode %llu i_nlink=%u",
                                  ea_inode->i_ino, ea_inode->i_nlink);

                        set_nlink(ea_inode, 1);
                        ext4_orphan_del(handle, ea_inode);
                }
        } else {
                if (ref_count == 0) {
                        WARN_ONCE(ea_inode->i_nlink != 1,
                                  "EA inode %llu i_nlink=%u",
                                  ea_inode->i_ino, ea_inode->i_nlink);

                        clear_nlink(ea_inode);
                        ext4_orphan_add(handle, ea_inode);
                }
        }

        ret = ext4_mark_iloc_dirty(handle, ea_inode, &iloc);
        if (ret)
                ext4_warning_inode(ea_inode,
                                   "ext4_mark_iloc_dirty() failed ret=%d", ret);
out:
        inode_unlock(ea_inode);
        return ret;
}

static int ext4_xattr_inode_inc_ref(handle_t *handle, struct inode *ea_inode)
{
        return ext4_xattr_inode_update_ref(handle, ea_inode, 1);
}

static int ext4_xattr_inode_dec_ref(handle_t *handle, struct inode *ea_inode)
{
        return ext4_xattr_inode_update_ref(handle, ea_inode, -1);
}

static int ext4_xattr_inode_inc_ref_all(handle_t *handle, struct inode *parent,
                                        struct ext4_xattr_entry *first)
{
        struct inode *ea_inode;
        struct ext4_xattr_entry *entry;
        struct ext4_xattr_entry *failed_entry;
        unsigned int ea_ino;
        int err, saved_err;

        for (entry = first; !IS_LAST_ENTRY(entry);
             entry = EXT4_XATTR_NEXT(entry)) {
                if (!entry->e_value_inum)
                        continue;
                ea_ino = le32_to_cpu(entry->e_value_inum);
                err = ext4_xattr_inode_iget(parent, ea_ino,
                                            le32_to_cpu(entry->e_hash),
                                            &ea_inode);
                if (err)
                        goto cleanup;
                err = ext4_xattr_inode_inc_ref(handle, ea_inode);
                if (err) {
                        ext4_warning_inode(ea_inode, "inc ref error %d", err);
                        ext4_put_ea_inode(ea_inode);
                        goto cleanup;
                }
                ext4_put_ea_inode(ea_inode);
        }
        return 0;

cleanup:
        saved_err = err;
        failed_entry = entry;

        for (entry = first; entry != failed_entry;
             entry = EXT4_XATTR_NEXT(entry)) {
                if (!entry->e_value_inum)
                        continue;
                ea_ino = le32_to_cpu(entry->e_value_inum);
                err = ext4_xattr_inode_iget(parent, ea_ino,
                                            le32_to_cpu(entry->e_hash),
                                            &ea_inode);
                if (err) {
                        ext4_warning(parent->i_sb,
                                     "cleanup ea_ino %u iget error %d", ea_ino,
                                     err);
                        continue;
                }
                err = ext4_xattr_inode_dec_ref(handle, ea_inode);
                if (err)
                        ext4_warning_inode(ea_inode, "cleanup dec ref error %d",
                                           err);
                ext4_put_ea_inode(ea_inode);
        }
        return saved_err;
}

static int ext4_xattr_restart_fn(handle_t *handle, struct inode *inode,
                        struct buffer_head *bh, bool block_csum, bool dirty)
{
        int error;

        if (bh && dirty) {
                if (block_csum)
                        ext4_xattr_block_csum_set(inode, bh);
                error = ext4_handle_dirty_metadata(handle, NULL, bh);
                if (error) {
                        ext4_warning(inode->i_sb, "Handle metadata (error %d)",
                                     error);
                        return error;
                }
        }
        return 0;
}

static void
ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent,
                             struct buffer_head *bh,
                             struct ext4_xattr_entry *first, bool block_csum,
                             int extra_credits, bool skip_quota)
{
        struct inode *ea_inode;
        struct ext4_xattr_entry *entry;
        struct ext4_iloc iloc = { .bh = NULL };
        bool dirty = false;
        unsigned int ea_ino;
        int err;
        int credits;
        void *end;

        if (block_csum)
                end = (void *)bh->b_data + bh->b_size;
        else {
                err = ext4_get_inode_loc(parent, &iloc);
                if (err) {
                        EXT4_ERROR_INODE(parent, "parent inode loc (error %d)", err);
                        return;
                }
                end = (void *)ext4_raw_inode(&iloc) + EXT4_SB(parent->i_sb)->s_inode_size;
        }

        /* One credit for dec ref on ea_inode, one for orphan list addition, */
        credits = 2 + extra_credits;

        for (entry = first; (void *)entry < end && !IS_LAST_ENTRY(entry);
             entry = EXT4_XATTR_NEXT(entry)) {
                if (!entry->e_value_inum)
                        continue;
                ea_ino = le32_to_cpu(entry->e_value_inum);
                err = ext4_xattr_inode_iget(parent, ea_ino,
                                            le32_to_cpu(entry->e_hash),
                                            &ea_inode);
                if (err)
                        continue;

                err = ext4_journal_ensure_credits_fn(handle, credits, credits,
                        ext4_free_metadata_revoke_credits(parent->i_sb, 1),
                        ext4_xattr_restart_fn(handle, parent, bh, block_csum,
                                              dirty));
                if (err < 0) {
                        ext4_warning_inode(ea_inode, "Ensure credits err=%d",
                                           err);
                        ext4_put_ea_inode(ea_inode);
                        continue;
                }
                if (err > 0) {
                        err = ext4_journal_get_write_access(handle,
                                        parent->i_sb, bh, EXT4_JTR_NONE);
                        if (err) {
                                ext4_warning_inode(ea_inode,
                                                "Re-get write access err=%d",
                                                err);
                                ext4_put_ea_inode(ea_inode);
                                continue;
                        }
                }

                err = ext4_xattr_inode_dec_ref(handle, ea_inode);
                if (err) {
                        ext4_warning_inode(ea_inode, "ea_inode dec ref err=%d",
                                           err);
                        ext4_put_ea_inode(ea_inode);
                        continue;
                }

                if (!skip_quota)
                        ext4_xattr_inode_free_quota(parent, ea_inode,
                                              le32_to_cpu(entry->e_value_size));

                /*
                 * Forget about ea_inode within the same transaction that
                 * decrements the ref count. This avoids duplicate decrements in
                 * case the rest of the work spills over to subsequent
                 * transactions.
                 */
                entry->e_value_inum = 0;
                entry->e_value_size = 0;

                ext4_put_ea_inode(ea_inode);
                dirty = true;
        }

        if (dirty) {
                /*
                 * Note that we are deliberately skipping csum calculation for
                 * the final update because we do not expect any journal
                 * restarts until xattr block is freed.
                 */

                err = ext4_handle_dirty_metadata(handle, NULL, bh);
                if (err)
                        ext4_warning_inode(parent,
                                           "handle dirty metadata err=%d", err);
        }

        brelse(iloc.bh);
}

/*
 * Release the xattr block BH: If the reference count is > 1, decrement it;
 * otherwise free the block.
 */
static void
ext4_xattr_release_block(handle_t *handle, struct inode *inode,
                         struct buffer_head *bh,
                         int extra_credits)
{
        struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode);
        u32 hash, ref;
        int error = 0;

        BUFFER_TRACE(bh, "get_write_access");
        error = ext4_journal_get_write_access(handle, inode->i_sb, bh,
                                              EXT4_JTR_NONE);
        if (error)
                goto out;

retry_ref:
        lock_buffer(bh);
        hash = le32_to_cpu(BHDR(bh)->h_hash);
        ref = le32_to_cpu(BHDR(bh)->h_refcount);
        if (ref == 1) {
                ea_bdebug(bh, "refcount now=0; freeing");
                /*
                 * This must happen under buffer lock for
                 * ext4_xattr_block_set() to reliably detect freed block
                 */
                if (ea_block_cache) {
                        struct mb_cache_entry *oe;

                        oe = mb_cache_entry_delete_or_get(ea_block_cache, hash,
                                                          bh->b_blocknr);
                        if (oe) {
                                unlock_buffer(bh);
                                mb_cache_entry_wait_unused(oe);
                                mb_cache_entry_put(ea_block_cache, oe);
                                goto retry_ref;
                        }
                }
                get_bh(bh);
                unlock_buffer(bh);

                if (ext4_has_feature_ea_inode(inode->i_sb))
                        ext4_xattr_inode_dec_ref_all(handle, inode, bh,
                                                     BFIRST(bh),
                                                     true /* block_csum */,
                                                     extra_credits,
                                                     true /* skip_quota */);
                ext4_free_blocks(handle, inode, bh, 0, 1,
                                 EXT4_FREE_BLOCKS_METADATA |
                                 EXT4_FREE_BLOCKS_FORGET);
        } else {
                ref--;
                BHDR(bh)->h_refcount = cpu_to_le32(ref);
                if (ref == EXT4_XATTR_REFCOUNT_MAX - 1) {
                        struct mb_cache_entry *ce;

                        if (ea_block_cache) {
                                ce = mb_cache_entry_get(ea_block_cache, hash,
                                                        bh->b_blocknr);
                                if (ce) {
                                        set_bit(MBE_REUSABLE_B, &ce->e_flags);
                                        mb_cache_entry_put(ea_block_cache, ce);
                                }
                        }
                }

                ext4_xattr_block_csum_set(inode, bh);
                /*
                 * Beware of this ugliness: Releasing of xattr block references
                 * from different inodes can race and so we have to protect
                 * from a race where someone else frees the block (and releases
                 * its journal_head) before we are done dirtying the buffer. In
                 * nojournal mode this race is harmless and we actually cannot
                 * call ext4_handle_dirty_metadata() with locked buffer as
                 * that function can call sync_dirty_buffer() so for that case
                 * we handle the dirtying after unlocking the buffer.
                 */
                if (ext4_handle_valid(handle))
                        error = ext4_handle_dirty_metadata(handle, inode, bh);
                unlock_buffer(bh);
                if (!ext4_handle_valid(handle))
                        error = ext4_handle_dirty_metadata(handle, inode, bh);
                if (IS_SYNC(inode))
                        ext4_handle_sync(handle);
                dquot_free_block(inode, EXT4_C2B(EXT4_SB(inode->i_sb), 1));
                ea_bdebug(bh, "refcount now=%d; releasing",
                          le32_to_cpu(BHDR(bh)->h_refcount));
        }
out:
        ext4_std_error(inode->i_sb, error);
        return;
}

/*
 * Find the available free space for EAs. This also returns the total number of
 * bytes used by EA entries.
 */
static size_t ext4_xattr_free_space(struct ext4_xattr_entry *last,
                                    size_t *min_offs, void *base, int *total)
{
        for (; !IS_LAST_ENTRY(last); last = EXT4_XATTR_NEXT(last)) {
                if (!last->e_value_inum && last->e_value_size) {
                        size_t offs = le16_to_cpu(last->e_value_offs);
                        if (offs < *min_offs)
                                *min_offs = offs;
                }
                if (total)
                        *total += EXT4_XATTR_LEN(last->e_name_len);
        }
        return (*min_offs - ((void *)last - base) - sizeof(__u32));
}

/*
 * Write the value of the EA in an inode.
 */
static int ext4_xattr_inode_write(handle_t *handle, struct inode *ea_inode,
                                  const void *buf, int bufsize)
{
        struct buffer_head *bh = NULL;
        unsigned long block = 0;
        int blocksize = ea_inode->i_sb->s_blocksize;
        int max_blocks = (bufsize + blocksize - 1) >> ea_inode->i_blkbits;
        int csize, wsize = 0;
        int ret = 0, ret2 = 0;
        int retries = 0;

retry:
        while (ret >= 0 && ret < max_blocks) {
                struct ext4_map_blocks map;
                map.m_lblk = block += ret;
                map.m_len = max_blocks -= ret;

                ret = ext4_map_blocks(handle, ea_inode, &map,
                                      EXT4_GET_BLOCKS_CREATE);
                if (ret <= 0) {
                        ext4_mark_inode_dirty(handle, ea_inode);
                        if (ret == -ENOSPC &&
                            ext4_should_retry_alloc(ea_inode->i_sb, &retries)) {
                                ret = 0;
                                goto retry;
                        }
                        break;
                }
        }

        if (ret < 0)
                return ret;

        block = 0;
        while (wsize < bufsize) {
                brelse(bh);
                csize = (bufsize - wsize) > blocksize ? blocksize :
                                                                bufsize - wsize;
                bh = ext4_getblk(handle, ea_inode, block, 0);
                if (IS_ERR(bh))
                        return PTR_ERR(bh);
                if (!bh) {
                        WARN_ON_ONCE(1);
                        EXT4_ERROR_INODE(ea_inode,
                                         "ext4_getblk() return bh = NULL");
                        return -EFSCORRUPTED;
                }
                ret = ext4_journal_get_write_access(handle, ea_inode->i_sb, bh,
                                                   EXT4_JTR_NONE);
                if (ret)
                        goto out;

                memcpy(bh->b_data, buf, csize);
                /*
                 * Zero out block tail to avoid writing uninitialized memory
                 * to disk.
                 */
                if (csize < blocksize)
                        memset(bh->b_data + csize, 0, blocksize - csize);
                set_buffer_uptodate(bh);
                ext4_handle_dirty_metadata(handle, ea_inode, bh);

                buf += csize;
                wsize += csize;
                block += 1;
        }

        inode_lock(ea_inode);
        i_size_write(ea_inode, wsize);
        ext4_update_i_disksize(ea_inode, wsize);
        inode_unlock(ea_inode);

        ret2 = ext4_mark_inode_dirty(handle, ea_inode);
        if (unlikely(ret2 && !ret))
                ret = ret2;

out:
        brelse(bh);

        return ret;
}

/*
 * Create an inode to store the value of a large EA.
 */
static struct inode *ext4_xattr_inode_create(handle_t *handle,
                                             struct inode *inode, u32 hash)
{
        struct inode *ea_inode = NULL;
        uid_t owner[2] = { i_uid_read(inode), i_gid_read(inode) };
        int err;

        if (inode->i_sb->s_root == NULL) {
                ext4_warning(inode->i_sb,
                             "refuse to create EA inode when umounting");
                WARN_ON(1);
                return ERR_PTR(-EINVAL);
        }

        /*
         * Let the next inode be the goal, so we try and allocate the EA inode
         * in the same group, or nearby one.
         */
        ea_inode = ext4_new_inode(handle, inode->i_sb->s_root->d_inode,
                                  S_IFREG | 0600, NULL, inode->i_ino + 1, owner,
                                  EXT4_EA_INODE_FL);
        if (!IS_ERR(ea_inode)) {
                ea_inode->i_op = &ext4_file_inode_operations;
                ea_inode->i_fop = &ext4_file_operations;
                ext4_set_aops(ea_inode);
                ext4_xattr_inode_set_class(ea_inode);
                unlock_new_inode(ea_inode);
                ext4_xattr_inode_set_ref(ea_inode, 1);
                ext4_xattr_inode_set_hash(ea_inode, hash);
                err = ext4_mark_inode_dirty(handle, ea_inode);
                if (!err)
                        err = ext4_inode_attach_jinode(ea_inode);
                if (err) {
                        if (ext4_xattr_inode_dec_ref(handle, ea_inode))
                                ext4_warning_inode(ea_inode,
                                        "cleanup dec ref error %d", err);
                        ext4_put_ea_inode(ea_inode);
                        return ERR_PTR(err);
                }

                /*
                 * Xattr inodes are shared therefore quota charging is performed
                 * at a higher level.
                 */
                dquot_free_inode(ea_inode);
                dquot_drop(ea_inode);
                inode_lock(ea_inode);
                ea_inode->i_flags |= S_NOQUOTA;
                inode_unlock(ea_inode);
        }

        return ea_inode;
}

static struct inode *
ext4_xattr_inode_cache_find(struct inode *inode, const void *value,
                            size_t value_len, u32 hash)
{
        struct inode *ea_inode;
        struct mb_cache_entry *ce;
        struct mb_cache *ea_inode_cache = EA_INODE_CACHE(inode);
        void *ea_data;

        if (!ea_inode_cache)
                return NULL;

        ce = mb_cache_entry_find_first(ea_inode_cache, hash);
        if (!ce)
                return NULL;

        WARN_ON_ONCE(ext4_handle_valid(journal_current_handle()) &&
                     !(current->flags & PF_MEMALLOC_NOFS));

        ea_data = kvmalloc(value_len, GFP_NOFS);
        if (!ea_data) {
                mb_cache_entry_put(ea_inode_cache, ce);
                return NULL;
        }

        while (ce) {
                ea_inode = ext4_iget(inode->i_sb, ce->e_value,
                                     EXT4_IGET_EA_INODE | EXT4_IGET_NOWAIT);
                if (IS_ERR(ea_inode))
                        goto next_entry;
                ext4_xattr_inode_set_class(ea_inode);
                if (i_size_read(ea_inode) == value_len &&
                    !ext4_xattr_inode_read(ea_inode, ea_data, value_len) &&
                    !ext4_xattr_inode_verify_hashes(ea_inode, NULL, ea_data,
                                                    value_len) &&
                    !memcmp(value, ea_data, value_len)) {
                        mb_cache_entry_touch(ea_inode_cache, ce);
                        mb_cache_entry_put(ea_inode_cache, ce);
                        kvfree(ea_data);
                        return ea_inode;
                }
                ext4_put_ea_inode(ea_inode);
        next_entry:
                ce = mb_cache_entry_find_next(ea_inode_cache, ce);
        }
        kvfree(ea_data);
        return NULL;
}

/*
 * Add value of the EA in an inode.
 */
static struct inode *ext4_xattr_inode_lookup_create(handle_t *handle,
                struct inode *inode, const void *value, size_t value_len)
{
        struct inode *ea_inode;
        u32 hash;
        int err;

        /* Account inode & space to quota even if sharing... */
        err = ext4_xattr_inode_alloc_quota(inode, value_len);
        if (err)
                return ERR_PTR(err);

        hash = ext4_xattr_inode_hash(EXT4_SB(inode->i_sb), value, value_len);
        ea_inode = ext4_xattr_inode_cache_find(inode, value, value_len, hash);
        if (ea_inode) {
                err = ext4_xattr_inode_inc_ref(handle, ea_inode);
                if (err)
                        goto out_err;
                return ea_inode;
        }

        /* Create an inode for the EA value */
        ea_inode = ext4_xattr_inode_create(handle, inode, hash);
        if (IS_ERR(ea_inode)) {
                ext4_xattr_inode_free_quota(inode, NULL, value_len);
                return ea_inode;
        }

        err = ext4_xattr_inode_write(handle, ea_inode, value, value_len);
        if (err) {
                if (ext4_xattr_inode_dec_ref(handle, ea_inode))
                        ext4_warning_inode(ea_inode, "cleanup dec ref error %d", err);
                goto out_err;
        }

        if (EA_INODE_CACHE(inode))
                mb_cache_entry_create(EA_INODE_CACHE(inode), GFP_NOFS, hash,
                                      ea_inode->i_ino, true /* reusable */);
        return ea_inode;
out_err:
        ext4_put_ea_inode(ea_inode);
        ext4_xattr_inode_free_quota(inode, NULL, value_len);
        return ERR_PTR(err);
}

/*
 * Reserve min(block_size/8, 1024) bytes for xattr entries/names if ea_inode
 * feature is enabled.
 */
#define EXT4_XATTR_BLOCK_RESERVE(inode)        min(i_blocksize(inode)/8, 1024U)

static int ext4_xattr_set_entry(struct ext4_xattr_info *i,
                                struct ext4_xattr_search *s,
                                handle_t *handle, struct inode *inode,
                                struct inode *new_ea_inode,
                                bool is_block)
{
        struct ext4_xattr_entry *last, *next;
        struct ext4_xattr_entry *here = s->here;
        size_t min_offs = s->end - s->base, name_len = strlen(i->name);
        int in_inode = i->in_inode;
        struct inode *old_ea_inode = NULL;
        size_t old_size, new_size;
        int ret;

        /* Space used by old and new values. */
        old_size = (!s->not_found && !here->e_value_inum) ?
                        EXT4_XATTR_SIZE(le32_to_cpu(here->e_value_size)) : 0;
        new_size = (i->value && !in_inode) ? EXT4_XATTR_SIZE(i->value_len) : 0;

        /*
         * Optimization for the simple case when old and new values have the
         * same padded sizes. Not applicable if external inodes are involved.
         */
        if (new_size && new_size == old_size) {
                size_t offs = le16_to_cpu(here->e_value_offs);
                void *val = s->base + offs;

                here->e_value_size = cpu_to_le32(i->value_len);
                if (i->value == EXT4_ZERO_XATTR_VALUE) {
                        memset(val, 0, new_size);
                } else {
                        memcpy(val, i->value, i->value_len);
                        /* Clear padding bytes. */
                        memset(val + i->value_len, 0, new_size - i->value_len);
                }
                goto update_hash;
        }

        /* Compute min_offs and last. */
        last = s->first;
        for (; !IS_LAST_ENTRY(last); last = next) {
                next = EXT4_XATTR_NEXT(last);
                if ((void *)next >= s->end) {
                        EXT4_ERROR_INODE(inode, "corrupted xattr entries");
                        ret = -EFSCORRUPTED;
                        goto out;
                }
                if (!last->e_value_inum && last->e_value_size) {
                        size_t offs = le16_to_cpu(last->e_value_offs);
                        if (offs < min_offs)
                                min_offs = offs;
                }
        }

        /* Check whether we have enough space. */
        if (i->value) {
                size_t free;

                free = min_offs - ((void *)last - s->base) - sizeof(__u32);
                if (!s->not_found)
                        free += EXT4_XATTR_LEN(name_len) + old_size;

                if (free < EXT4_XATTR_LEN(name_len) + new_size) {
                        ret = -ENOSPC;
                        goto out;
                }

                /*
                 * If storing the value in an external inode is an option,
                 * reserve space for xattr entries/names in the external
                 * attribute block so that a long value does not occupy the
                 * whole space and prevent further entries being added.
                 */
                if (ext4_has_feature_ea_inode(inode->i_sb) &&
                    new_size && is_block &&
                    (min_offs + old_size - new_size) <
                                        EXT4_XATTR_BLOCK_RESERVE(inode)) {
                        ret = -ENOSPC;
                        goto out;
                }
        }

        /*
         * Getting access to old and new ea inodes is subject to failures.
         * Finish that work before doing any modifications to the xattr data.
         */
        if (!s->not_found && here->e_value_inum) {
                ret = ext4_xattr_inode_iget(inode,
                                            le32_to_cpu(here->e_value_inum),
                                            le32_to_cpu(here->e_hash),
                                            &old_ea_inode);
                if (ret) {
                        old_ea_inode = NULL;
                        goto out;
                }

                /* We are ready to release ref count on the old_ea_inode. */
                ret = ext4_xattr_inode_dec_ref(handle, old_ea_inode);
                if (ret)
                        goto out;

                ext4_xattr_inode_free_quota(inode, old_ea_inode,
                                            le32_to_cpu(here->e_value_size));
        }

        /* No failures allowed past this point. */

        if (!s->not_found && here->e_value_size && !here->e_value_inum) {
                /* Remove the old value. */
                void *first_val = s->base + min_offs;
                size_t offs = le16_to_cpu(here->e_value_offs);
                void *val = s->base + offs;

                memmove(first_val + old_size, first_val, val - first_val);
                memset(first_val, 0, old_size);
                min_offs += old_size;

                /* Adjust all value offsets. */
                last = s->first;
                while (!IS_LAST_ENTRY(last)) {
                        size_t o = le16_to_cpu(last->e_value_offs);

                        if (!last->e_value_inum &&
                            last->e_value_size && o < offs)
                                last->e_value_offs = cpu_to_le16(o + old_size);
                        last = EXT4_XATTR_NEXT(last);
                }
        }

        if (!i->value) {
                /* Remove old name. */
                size_t size = EXT4_XATTR_LEN(name_len);

                last = ENTRY((void *)last - size);
                memmove(here, (void *)here + size,
                        (void *)last - (void *)here + sizeof(__u32));
                memset(last, 0, size);

                /*
                 * Update i_inline_off - moved ibody region might contain
                 * system.data attribute.  Handling a failure here won't
                 * cause other complications for setting an xattr.
                 */
                if (!is_block && ext4_has_inline_data(inode)) {
                        ret = ext4_find_inline_data_nolock(inode);
                        if (ret) {
                                ext4_warning_inode(inode,
                                        "unable to update i_inline_off");
                                goto out;
                        }
                }
        } else if (s->not_found) {
                /* Insert new name. */
                size_t size = EXT4_XATTR_LEN(name_len);
                size_t rest = (void *)last - (void *)here + sizeof(__u32);

                memmove((void *)here + size, here, rest);
                memset(here, 0, size);
                here->e_name_index = i->name_index;
                here->e_name_len = name_len;
                memcpy(here->e_name, i->name, name_len);
        } else {
                /* This is an update, reset value info. */
                here->e_value_inum = 0;
                here->e_value_offs = 0;
                here->e_value_size = 0;
        }

        if (i->value) {
                /* Insert new value. */
                if (in_inode) {
                        here->e_value_inum = cpu_to_le32(new_ea_inode->i_ino);
                } else if (i->value_len) {
                        void *val = s->base + min_offs - new_size;

                        here->e_value_offs = cpu_to_le16(min_offs - new_size);
                        if (i->value == EXT4_ZERO_XATTR_VALUE) {
                                memset(val, 0, new_size);
                        } else {
                                memcpy(val, i->value, i->value_len);
                                /* Clear padding bytes. */
                                memset(val + i->value_len, 0,
                                       new_size - i->value_len);
                        }
                }
                here->e_value_size = cpu_to_le32(i->value_len);
        }

update_hash:
        if (i->value) {
                __le32 hash = 0;

                /* Entry hash calculation. */
                if (in_inode) {
                        __le32 crc32c_hash;

                        /*
                         * Feed crc32c hash instead of the raw value for entry
                         * hash calculation. This is to avoid walking
                         * potentially long value buffer again.
                         */
                        crc32c_hash = cpu_to_le32(
                                       ext4_xattr_inode_get_hash(new_ea_inode));
                        hash = ext4_xattr_hash_entry(here->e_name,
                                                     here->e_name_len,
                                                     &crc32c_hash, 1);
                } else if (is_block) {
                        __le32 *value = s->base + le16_to_cpu(
                                                        here->e_value_offs);

                        hash = ext4_xattr_hash_entry(here->e_name,
                                                     here->e_name_len, value,
                                                     new_size >> 2);
                }
                here->e_hash = hash;
        }

        if (is_block)
                ext4_xattr_rehash((struct ext4_xattr_header *)s->base);

        ret = 0;
out:
        ext4_put_ea_inode(old_ea_inode);
        return ret;
}

struct ext4_xattr_block_find {
        struct ext4_xattr_search s;
        struct buffer_head *bh;
};

static int
ext4_xattr_block_find(struct inode *inode, struct ext4_xattr_info *i,
                      struct ext4_xattr_block_find *bs)
{
        struct super_block *sb = inode->i_sb;
        int error;

        ea_idebug(inode, "name=%d.%s, value=%p, value_len=%ld",
                  i->name_index, i->name, i->value, (long)i->value_len);

        if (EXT4_I(inode)->i_file_acl) {
                /* The inode already has an extended attribute block. */
                bs->bh = ext4_sb_bread(sb, EXT4_I(inode)->i_file_acl, REQ_PRIO);
                if (IS_ERR(bs->bh)) {
                        error = PTR_ERR(bs->bh);
                        bs->bh = NULL;
                        return error;
                }
                ea_bdebug(bs->bh, "b_count=%d, refcount=%d",
                        atomic_read(&(bs->bh->b_count)),
                        le32_to_cpu(BHDR(bs->bh)->h_refcount));
                error = ext4_xattr_check_block(inode, bs->bh);
                if (error)
                        return error;
                /* Find the named attribute. */
                bs->s.base = BHDR(bs->bh);
                bs->s.first = BFIRST(bs->bh);
                bs->s.end = bs->bh->b_data + bs->bh->b_size;
                bs->s.here = bs->s.first;
                error = xattr_find_entry(inode, &bs->s.here, bs->s.end,
                                         i->name_index, i->name, 1);
                if (error && error != -ENODATA)
                        return error;
                bs->s.not_found = error;
        }
        return 0;
}

static int
ext4_xattr_block_set(handle_t *handle, struct inode *inode,
                     struct ext4_xattr_info *i,
                     struct ext4_xattr_block_find *bs)
{
        struct super_block *sb = inode->i_sb;
        struct buffer_head *new_bh = NULL;
        struct ext4_xattr_search s_copy = bs->s;
        struct ext4_xattr_search *s = &s_copy;
        struct mb_cache_entry *ce = NULL;
        int error = 0;
        struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode);
        struct inode *ea_inode = NULL, *tmp_inode;
        size_t old_ea_inode_quota = 0;
        unsigned int ea_ino;

#define header(x) ((struct ext4_xattr_header *)(x))

        /* If we need EA inode, prepare it before locking the buffer */
        if (i->value && i->in_inode) {
                WARN_ON_ONCE(!i->value_len);

                ea_inode = ext4_xattr_inode_lookup_create(handle, inode,
                                        i->value, i->value_len);
                if (IS_ERR(ea_inode)) {
                        error = PTR_ERR(ea_inode);
                        ea_inode = NULL;
                        goto cleanup;
                }
        }

        if (s->base) {
                int offset = (char *)s->here - bs->bh->b_data;

                BUFFER_TRACE(bs->bh, "get_write_access");
                error = ext4_journal_get_write_access(handle, sb, bs->bh,
                                                      EXT4_JTR_NONE);
                if (error)
                        goto cleanup;

                lock_buffer(bs->bh);

                if (header(s->base)->h_refcount == cpu_to_le32(1)) {
                        __u32 hash = le32_to_cpu(BHDR(bs->bh)->h_hash);

                        /*
                         * This must happen under buffer lock for
                         * ext4_xattr_block_set() to reliably detect modified
                         * block
                         */
                        if (ea_block_cache) {
                                struct mb_cache_entry *oe;

                                oe = mb_cache_entry_delete_or_get(ea_block_cache,
                                        hash, bs->bh->b_blocknr);
                                if (oe) {
                                        /*
                                         * Xattr block is getting reused. Leave
                                         * it alone.
                                         */
                                        mb_cache_entry_put(ea_block_cache, oe);
                                        goto clone_block;
                                }
                        }
                        ea_bdebug(bs->bh, "modifying in-place");
                        error = ext4_xattr_set_entry(i, s, handle, inode,
                                             ea_inode, true /* is_block */);
                        ext4_xattr_block_csum_set(inode, bs->bh);
                        unlock_buffer(bs->bh);
                        if (error == -EFSCORRUPTED)
                                goto bad_block;
                        if (!error)
                                error = ext4_handle_dirty_metadata(handle,
                                                                   inode,
                                                                   bs->bh);
                        if (error)
                                goto cleanup;
                        goto inserted;
                }
clone_block:
                unlock_buffer(bs->bh);
                ea_bdebug(bs->bh, "cloning");
                s->base = kmemdup(BHDR(bs->bh), bs->bh->b_size, GFP_NOFS);
                error = -ENOMEM;
                if (s->base == NULL)
                        goto cleanup;
                s->first = ENTRY(header(s->base)+1);
                header(s->base)->h_refcount = cpu_to_le32(1);
                s->here = ENTRY(s->base + offset);
                s->end = s->base + bs->bh->b_size;

                /*
                 * If existing entry points to an xattr inode, we need
                 * to prevent ext4_xattr_set_entry() from decrementing
                 * ref count on it because the reference belongs to the
                 * original block. In this case, make the entry look
                 * like it has an empty value.
                 */
                if (!s->not_found && s->here->e_value_inum) {
                        ea_ino = le32_to_cpu(s->here->e_value_inum);
                        error = ext4_xattr_inode_iget(inode, ea_ino,
                                      le32_to_cpu(s->here->e_hash),
                                      &tmp_inode);
                        if (error)
                                goto cleanup;

                        if (!ext4_test_inode_state(tmp_inode,
                                        EXT4_STATE_LUSTRE_EA_INODE)) {
                                /*
                                 * Defer quota free call for previous
                                 * inode until success is guaranteed.
                                 */
                                old_ea_inode_quota = le32_to_cpu(
                                                s->here->e_value_size);
                        }
                        ext4_put_ea_inode(tmp_inode);

                        s->here->e_value_inum = 0;
                        s->here->e_value_size = 0;
                }
        } else {
                /* Allocate a buffer where we construct the new block. */
                s->base = kzalloc(sb->s_blocksize, GFP_NOFS);
                error = -ENOMEM;
                if (s->base == NULL)
                        goto cleanup;
                header(s->base)->h_magic = cpu_to_le32(EXT4_XATTR_MAGIC);
                header(s->base)->h_blocks = cpu_to_le32(1);
                header(s->base)->h_refcount = cpu_to_le32(1);
                s->first = ENTRY(header(s->base)+1);
                s->here = ENTRY(header(s->base)+1);
                s->end = s->base + sb->s_blocksize;
        }

        error = ext4_xattr_set_entry(i, s, handle, inode, ea_inode,
                                     true /* is_block */);
        if (error == -EFSCORRUPTED)
                goto bad_block;
        if (error)
                goto cleanup;

inserted:
        if (!IS_LAST_ENTRY(s->first)) {
                new_bh = ext4_xattr_block_cache_find(inode, header(s->base), &ce);
                if (IS_ERR(new_bh)) {
                        error = PTR_ERR(new_bh);
                        new_bh = NULL;
                        goto cleanup;
                }

                if (new_bh) {
                        /* We found an identical block in the cache. */
                        if (new_bh == bs->bh)
                                ea_bdebug(new_bh, "keeping");
                        else {
                                u32 ref;

#ifdef EXT4_XATTR_DEBUG
                                WARN_ON_ONCE(dquot_initialize_needed(inode));
#endif
                                /* The old block is released after updating
                                   the inode. */
                                error = dquot_alloc_block(inode,
                                                EXT4_C2B(EXT4_SB(sb), 1));
                                if (error)
                                        goto cleanup;
                                BUFFER_TRACE(new_bh, "get_write_access");
                                error = ext4_journal_get_write_access(
                                                handle, sb, new_bh,
                                                EXT4_JTR_NONE);
                                if (error)
                                        goto cleanup_dquot;
                                lock_buffer(new_bh);
                                /*
                                 * We have to be careful about races with
                                 * adding references to xattr block. Once we
                                 * hold buffer lock xattr block's state is
                                 * stable so we can check the additional
                                 * reference fits.
                                 */
                                ref = le32_to_cpu(BHDR(new_bh)->h_refcount);
                                if (ref >= EXT4_XATTR_REFCOUNT_MAX) {
                                        /*
                                         * Undo everything and check mbcache
                                         * again.
                                         */
                                        clear_bit(MBE_REUSABLE_B, &ce->e_flags);
                                        unlock_buffer(new_bh);
                                        dquot_free_block(inode,
                                                         EXT4_C2B(EXT4_SB(sb),
                                                                  1));
                                        brelse(new_bh);
                                        mb_cache_entry_put(ea_block_cache, ce);
                                        ce = NULL;
                                        new_bh = NULL;
                                        goto inserted;
                                }
                                ref++;
                                BHDR(new_bh)->h_refcount = cpu_to_le32(ref);
                                if (ref == EXT4_XATTR_REFCOUNT_MAX)
                                        clear_bit(MBE_REUSABLE_B, &ce->e_flags);
                                ea_bdebug(new_bh, "reusing; refcount now=%d",
                                          ref);
                                ext4_xattr_block_csum_set(inode, new_bh);
                                unlock_buffer(new_bh);
                                error = ext4_handle_dirty_metadata(handle,
                                                                   inode,
                                                                   new_bh);
                                if (error)
                                        goto cleanup_dquot;
                        }
                        mb_cache_entry_touch(ea_block_cache, ce);
                        mb_cache_entry_put(ea_block_cache, ce);
                        ce = NULL;
                } else if (bs->bh && s->base == bs->bh->b_data) {
                        /* We were modifying this block in-place. */
                        ea_bdebug(bs->bh, "keeping this block");
                        ext4_xattr_block_cache_insert(ea_block_cache, bs->bh);
                        new_bh = bs->bh;
                        get_bh(new_bh);
                } else {
                        /* We need to allocate a new block */
                        ext4_fsblk_t goal, block;

#ifdef EXT4_XATTR_DEBUG
                        WARN_ON_ONCE(dquot_initialize_needed(inode));
#endif
                        goal = ext4_group_first_block_no(sb,
                                                EXT4_I(inode)->i_block_group);
                        block = ext4_new_meta_blocks(handle, inode, goal, 0,
                                                     NULL, &error);
                        if (error)
                                goto cleanup;

                        ea_idebug(inode, "creating block %llu",
                                  (unsigned long long)block);

                        new_bh = sb_getblk(sb, block);
                        if (unlikely(!new_bh)) {
                                error = -ENOMEM;
getblk_failed:
                                ext4_free_blocks(handle, inode, NULL, block, 1,
                                                 EXT4_FREE_BLOCKS_METADATA);
                                goto cleanup;
                        }
                        error = ext4_xattr_inode_inc_ref_all(handle, inode,
                                                      ENTRY(header(s->base)+1));
                        if (error)
                                goto getblk_failed;
                        if (ea_inode) {
                                /* Drop the extra ref on ea_inode. */
                                error = ext4_xattr_inode_dec_ref(handle,
                                                                 ea_inode);
                                if (error)
                                        ext4_warning_inode(ea_inode,
                                                           "dec ref error=%d",
                                                           error);
                                ext4_put_ea_inode(ea_inode);
                                ea_inode = NULL;
                        }

                        lock_buffer(new_bh);
                        error = ext4_journal_get_create_access(handle, sb,
                                                        new_bh, EXT4_JTR_NONE);
                        if (error) {
                                unlock_buffer(new_bh);
                                error = -EIO;
                                goto getblk_failed;
                        }
                        memcpy(new_bh->b_data, s->base, new_bh->b_size);
                        ext4_xattr_block_csum_set(inode, new_bh);
                        set_buffer_uptodate(new_bh);
                        unlock_buffer(new_bh);
                        ext4_xattr_block_cache_insert(ea_block_cache, new_bh);
                        error = ext4_handle_dirty_metadata(handle, inode,
                                                           new_bh);
                        if (error)
                                goto cleanup;
                }
        }

        if (old_ea_inode_quota)
                ext4_xattr_inode_free_quota(inode, NULL, old_ea_inode_quota);

        /* Update the inode. */
        EXT4_I(inode)->i_file_acl = new_bh ? new_bh->b_blocknr : 0;

        /* Drop the previous xattr block. */
        if (bs->bh && bs->bh != new_bh) {
                ext4_xattr_release_block(handle, inode, bs->bh,
                                         0 /* extra_credits */);
        }
        error = 0;

cleanup:
        if (ea_inode) {
                if (error) {
                        int error2;

                        error2 = ext4_xattr_inode_dec_ref(handle, ea_inode);
                        if (error2)
                                ext4_warning_inode(ea_inode, "dec ref error=%d",
                                                   error2);
                        ext4_xattr_inode_free_quota(inode, ea_inode,
                                                    i_size_read(ea_inode));
                }
                ext4_put_ea_inode(ea_inode);
        }
        if (ce)
                mb_cache_entry_put(ea_block_cache, ce);
        brelse(new_bh);
        if (!(bs->bh && s->base == bs->bh->b_data))
                kfree(s->base);

        return error;

cleanup_dquot:
        dquot_free_block(inode, EXT4_C2B(EXT4_SB(sb), 1));
        goto cleanup;

bad_block:
        EXT4_ERROR_INODE(inode, "bad block %llu",
                         EXT4_I(inode)->i_file_acl);
        goto cleanup;

#undef header
}

int ext4_xattr_ibody_find(struct inode *inode, struct ext4_xattr_info *i,
                          struct ext4_xattr_ibody_find *is)
{
        struct ext4_xattr_ibody_header *header;
        struct ext4_inode *raw_inode;
        int error;

        if (!EXT4_INODE_HAS_XATTR_SPACE(inode))
                return 0;

        raw_inode = ext4_raw_inode(&is->iloc);
        header = IHDR(inode, raw_inode);
        is->s.base = is->s.first = IFIRST(header);
        is->s.here = is->s.first;
        is->s.end = ITAIL(inode, raw_inode);
        if (ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {
                /* Find the named attribute. */
                error = xattr_find_entry(inode, &is->s.here, is->s.end,
                                         i->name_index, i->name, 0);
                if (error && error != -ENODATA)
                        return error;
                is->s.not_found = error;
        }
        return 0;
}

int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode,
                                struct ext4_xattr_info *i,
                                struct ext4_xattr_ibody_find *is)
{
        struct ext4_xattr_ibody_header *header;
        struct ext4_xattr_search *s = &is->s;
        struct inode *ea_inode = NULL;
        int error;

        if (!EXT4_INODE_HAS_XATTR_SPACE(inode))
                return -ENOSPC;

        /* If we need EA inode, prepare it before locking the buffer */
        if (i->value && i->in_inode) {
                WARN_ON_ONCE(!i->value_len);

                ea_inode = ext4_xattr_inode_lookup_create(handle, inode,
                                        i->value, i->value_len);
                if (IS_ERR(ea_inode))
                        return PTR_ERR(ea_inode);
        }
        error = ext4_xattr_set_entry(i, s, handle, inode, ea_inode,
                                     false /* is_block */);
        if (error) {
                if (ea_inode) {
                        int error2;

                        error2 = ext4_xattr_inode_dec_ref(handle, ea_inode);
                        if (error2)
                                ext4_warning_inode(ea_inode, "dec ref error=%d",
                                                   error2);

                        ext4_xattr_inode_free_quota(inode, ea_inode,
                                                    i_size_read(ea_inode));
                        ext4_put_ea_inode(ea_inode);
                }
                return error;
        }
        header = IHDR(inode, ext4_raw_inode(&is->iloc));
        if (!IS_LAST_ENTRY(s->first)) {
                header->h_magic = cpu_to_le32(EXT4_XATTR_MAGIC);
                ext4_set_inode_state(inode, EXT4_STATE_XATTR);
        } else {
                header->h_magic = cpu_to_le32(0);
                ext4_clear_inode_state(inode, EXT4_STATE_XATTR);
        }
        ext4_put_ea_inode(ea_inode);
        return 0;
}

static int ext4_xattr_value_same(struct ext4_xattr_search *s,
                                 struct ext4_xattr_info *i)
{
        void *value;

        /* When e_value_inum is set the value is stored externally. */
        if (s->here->e_value_inum)
                return 0;
        if (le32_to_cpu(s->here->e_value_size) != i->value_len)
                return 0;
        value = ((void *)s->base) + le16_to_cpu(s->here->e_value_offs);
        return !memcmp(value, i->value, i->value_len);
}

static struct buffer_head *ext4_xattr_get_block(struct inode *inode)
{
        struct buffer_head *bh;
        int error;

        if (!EXT4_I(inode)->i_file_acl)
                return NULL;
        bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO);
        if (IS_ERR(bh))
                return bh;
        error = ext4_xattr_check_block(inode, bh);
        if (error) {
                brelse(bh);
                return ERR_PTR(error);
        }
        return bh;
}

/*
 * ext4_xattr_set_handle()
 *
 * Create, replace or remove an extended attribute for this inode.  Value
 * is NULL to remove an existing extended attribute, and non-NULL to
 * either replace an existing extended attribute, or create a new extended
 * attribute. The flags XATTR_REPLACE and XATTR_CREATE
 * specify that an extended attribute must exist and must not exist
 * previous to the call, respectively.
 *
 * Returns 0, or a negative error number on failure.
 */
int
ext4_xattr_set_handle(handle_t *handle, struct inode *inode, int name_index,
                      const char *name, const void *value, size_t value_len,
                      int flags)
{
        struct ext4_xattr_info i = {
                .name_index = name_index,
                .name = name,
                .value = value,
                .value_len = value_len,
                .in_inode = 0,
        };
        struct ext4_xattr_ibody_find is = {
                .s = { .not_found = -ENODATA, },
        };
        struct ext4_xattr_block_find bs = {
                .s = { .not_found = -ENODATA, },
        };
        int no_expand;
        int error;

        if (!name)
                return -EINVAL;
        if (strlen(name) > 255)
                return -ERANGE;

        ext4_write_lock_xattr(inode, &no_expand);

        /* Check journal credits under write lock. */
        if (ext4_handle_valid(handle)) {
                struct buffer_head *bh;
                int credits;

                bh = ext4_xattr_get_block(inode);
                if (IS_ERR(bh)) {
                        error = PTR_ERR(bh);
                        goto cleanup;
                }

                credits = __ext4_xattr_set_credits(inode->i_sb, inode, bh,
                                                   value_len,
                                                   flags & XATTR_CREATE);
                brelse(bh);

                if (jbd2_handle_buffer_credits(handle) < credits) {
                        error = -ENOSPC;
                        goto cleanup;
                }
                WARN_ON_ONCE(!(current->flags & PF_MEMALLOC_NOFS));
        }

        error = ext4_reserve_inode_write(handle, inode, &is.iloc);
        if (error)
                goto cleanup;

        if (ext4_test_inode_state(inode, EXT4_STATE_NEW)) {
                struct ext4_inode *raw_inode = ext4_raw_inode(&is.iloc);
                memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
                ext4_clear_inode_state(inode, EXT4_STATE_NEW);
        }

        error = ext4_xattr_ibody_find(inode, &i, &is);
        if (error)
                goto cleanup;
        if (is.s.not_found)
                error = ext4_xattr_block_find(inode, &i, &bs);
        if (error)
                goto cleanup;
        if (is.s.not_found && bs.s.not_found) {
                error = -ENODATA;
                if (flags & XATTR_REPLACE)
                        goto cleanup;
                error = 0;
                if (!value)
                        goto cleanup;
        } else {
                error = -EEXIST;
                if (flags & XATTR_CREATE)
                        goto cleanup;
        }

        if (!value) {
                if (!is.s.not_found)
                        error = ext4_xattr_ibody_set(handle, inode, &i, &is);
                else if (!bs.s.not_found)
                        error = ext4_xattr_block_set(handle, inode, &i, &bs);
        } else {
                error = 0;
                /* Xattr value did not change? Save us some work and bail out */
                if (!is.s.not_found && ext4_xattr_value_same(&is.s, &i))
                        goto cleanup;
                if (!bs.s.not_found && ext4_xattr_value_same(&bs.s, &i))
                        goto cleanup;

                if (ext4_has_feature_ea_inode(inode->i_sb) &&
                    (EXT4_XATTR_SIZE(i.value_len) >
                        EXT4_XATTR_MIN_LARGE_EA_SIZE(inode->i_sb->s_blocksize)))
                        i.in_inode = 1;
retry_inode:
                error = ext4_xattr_ibody_set(handle, inode, &i, &is);
                if (!error && !bs.s.not_found) {
                        i.value = NULL;
                        error = ext4_xattr_block_set(handle, inode, &i, &bs);
                } else if (error == -ENOSPC) {
                        if (EXT4_I(inode)->i_file_acl && !bs.s.base) {
                                brelse(bs.bh);
                                bs.bh = NULL;
                                error = ext4_xattr_block_find(inode, &i, &bs);
                                if (error)
                                        goto cleanup;
                        }
                        error = ext4_xattr_block_set(handle, inode, &i, &bs);
                        if (!error && !is.s.not_found) {
                                i.value = NULL;
                                error = ext4_xattr_ibody_set(handle, inode, &i,
                                                             &is);
                        } else if (error == -ENOSPC) {
                                /*
                                 * Xattr does not fit in the block, store at
                                 * external inode if possible.
                                 */
                                if (ext4_has_feature_ea_inode(inode->i_sb) &&
                                    i.value_len && !i.in_inode) {
                                        i.in_inode = 1;
                                        goto retry_inode;
                                }
                        }
                }
        }
        if (!error) {
                ext4_xattr_update_super_block(handle, inode->i_sb);
                inode_set_ctime_current(inode);
                inode_inc_iversion(inode);
                if (!value)
                        no_expand = 0;
                error = ext4_mark_iloc_dirty(handle, inode, &is.iloc);
                /*
                 * The bh is consumed by ext4_mark_iloc_dirty, even with
                 * error != 0.
                 */
                is.iloc.bh = NULL;
                if (IS_SYNC(inode))
                        ext4_handle_sync(handle);
        }
        ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, handle);

cleanup:
        brelse(is.iloc.bh);
        brelse(bs.bh);
        ext4_write_unlock_xattr(inode, &no_expand);
        return error;
}

int ext4_xattr_set_credits(struct inode *inode, size_t value_len,
                           bool is_create, int *credits)
{
        struct buffer_head *bh;
        int err;

        *credits = 0;

        if (!EXT4_SB(inode->i_sb)->s_journal)
                return 0;

        down_read(&EXT4_I(inode)->xattr_sem);

        bh = ext4_xattr_get_block(inode);
        if (IS_ERR(bh)) {
                err = PTR_ERR(bh);
        } else {
                *credits = __ext4_xattr_set_credits(inode->i_sb, inode, bh,
                                                    value_len, is_create);
                brelse(bh);
                err = 0;
        }

        up_read(&EXT4_I(inode)->xattr_sem);
        return err;
}

/*
 * ext4_xattr_set()
 *
 * Like ext4_xattr_set_handle, but start from an inode. This extended
 * attribute modification is a filesystem transaction by itself.
 *
 * Returns 0, or a negative error number on failure.
 */
int
ext4_xattr_set(struct inode *inode, int name_index, const char *name,
               const void *value, size_t value_len, int flags)
{
        handle_t *handle;
        struct super_block *sb = inode->i_sb;
        int error, retries = 0;
        int credits;

        error = dquot_initialize(inode);
        if (error)
                return error;

retry:
        error = ext4_xattr_set_credits(inode, value_len, flags & XATTR_CREATE,
                                       &credits);
        if (error)
                return error;

        handle = ext4_journal_start(inode, EXT4_HT_XATTR, credits);
        if (IS_ERR(handle)) {
                error = PTR_ERR(handle);
        } else {
                int error2;

                error = ext4_xattr_set_handle(handle, inode, name_index, name,
                                              value, value_len, flags);
                ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR,
                                        handle);
                error2 = ext4_journal_stop(handle);
                if (error == -ENOSPC &&
                    ext4_should_retry_alloc(sb, &retries))
                        goto retry;
                if (error == 0)
                        error = error2;
        }

        return error;
}

/*
 * Shift the EA entries in the inode to create space for the increased
 * i_extra_isize.
 */
static void ext4_xattr_shift_entries(struct ext4_xattr_entry *entry,
                                     int value_offs_shift, void *to,
                                     void *from, size_t n)
{
        struct ext4_xattr_entry *last = entry;
        int new_offs;

        /* We always shift xattr headers further thus offsets get lower */
        BUG_ON(value_offs_shift > 0);

        /* Adjust the value offsets of the entries */
        for (; !IS_LAST_ENTRY(last); last = EXT4_XATTR_NEXT(last)) {
                if (!last->e_value_inum && last->e_value_size) {
                        new_offs = le16_to_cpu(last->e_value_offs) +
                                                        value_offs_shift;
                        last->e_value_offs = cpu_to_le16(new_offs);
                }
        }
        /* Shift the entries by n bytes */
        memmove(to, from, n);
}

/*
 * Move xattr pointed to by 'entry' from inode into external xattr block
 */
static int ext4_xattr_move_to_block(handle_t *handle, struct inode *inode,
                                    struct ext4_inode *raw_inode,
                                    struct ext4_xattr_entry *entry)
{
        struct ext4_xattr_ibody_find *is = NULL;
        struct ext4_xattr_block_find *bs = NULL;
        char *buffer = NULL, *b_entry_name = NULL;
        size_t value_size = le32_to_cpu(entry->e_value_size);
        struct ext4_xattr_info i = {
                .value = NULL,
                .value_len = 0,
                .name_index = entry->e_name_index,
                .in_inode = !!entry->e_value_inum,
        };
        struct ext4_xattr_ibody_header *header = IHDR(inode, raw_inode);
        int needs_kvfree = 0;
        int error;

        is = kzalloc_obj(struct ext4_xattr_ibody_find, GFP_NOFS);
        bs = kzalloc_obj(struct ext4_xattr_block_find, GFP_NOFS);
        b_entry_name = kmalloc(entry->e_name_len + 1, GFP_NOFS);
        if (!is || !bs || !b_entry_name) {
                error = -ENOMEM;
                goto out;
        }

        is->s.not_found = -ENODATA;
        bs->s.not_found = -ENODATA;
        is->iloc.bh = NULL;
        bs->bh = NULL;

        /* Save the entry name and the entry value */
        if (entry->e_value_inum) {
                buffer = kvmalloc(value_size, GFP_NOFS);
                if (!buffer) {
                        error = -ENOMEM;
                        goto out;
                }
                needs_kvfree = 1;
                error = ext4_xattr_inode_get(inode, entry, buffer, value_size);
                if (error)
                        goto out;
        } else {
                size_t value_offs = le16_to_cpu(entry->e_value_offs);
                buffer = (void *)IFIRST(header) + value_offs;
        }

        memcpy(b_entry_name, entry->e_name, entry->e_name_len);
        b_entry_name[entry->e_name_len] = '\0';
        i.name = b_entry_name;

        error = ext4_get_inode_loc(inode, &is->iloc);
        if (error)
                goto out;

        error = ext4_xattr_ibody_find(inode, &i, is);
        if (error)
                goto out;

        i.value = buffer;
        i.value_len = value_size;
        error = ext4_xattr_block_find(inode, &i, bs);
        if (error)
                goto out;

        /* Move ea entry from the inode into the block */
        error = ext4_xattr_block_set(handle, inode, &i, bs);
        if (error)
                goto out;

        /* Remove the chosen entry from the inode */
        i.value = NULL;
        i.value_len = 0;
        error = ext4_xattr_ibody_set(handle, inode, &i, is);

out:
        kfree(b_entry_name);
        if (needs_kvfree && buffer)
                kvfree(buffer);
        if (is)
                brelse(is->iloc.bh);
        if (bs)
                brelse(bs->bh);
        kfree(is);
        kfree(bs);

        return error;
}

static int ext4_xattr_make_inode_space(handle_t *handle, struct inode *inode,
                                       struct ext4_inode *raw_inode,
                                       int isize_diff, size_t ifree,
                                       size_t bfree, int *total_ino)
{
        struct ext4_xattr_ibody_header *header = IHDR(inode, raw_inode);
        struct ext4_xattr_entry *small_entry;
        struct ext4_xattr_entry *entry;
        struct ext4_xattr_entry *last;
        unsigned int entry_size;        /* EA entry size */
        unsigned int total_size;        /* EA entry size + value size */
        unsigned int min_total_size;
        int error;

        while (isize_diff > ifree) {
                entry = NULL;
                small_entry = NULL;
                min_total_size = ~0U;
                last = IFIRST(header);
                /* Find the entry best suited to be pushed into EA block */
                for (; !IS_LAST_ENTRY(last); last = EXT4_XATTR_NEXT(last)) {
                        /* never move system.data out of the inode */
                        if ((last->e_name_len == 4) &&
                            (last->e_name_index == EXT4_XATTR_INDEX_SYSTEM) &&
                            !memcmp(last->e_name, "data", 4))
                                continue;
                        total_size = EXT4_XATTR_LEN(last->e_name_len);
                        if (!last->e_value_inum)
                                total_size += EXT4_XATTR_SIZE(
                                               le32_to_cpu(last->e_value_size));
                        if (total_size <= bfree &&
                            total_size < min_total_size) {
                                if (total_size + ifree < isize_diff) {
                                        small_entry = last;
                                } else {
                                        entry = last;
                                        min_total_size = total_size;
                                }
                        }
                }

                if (entry == NULL) {
                        if (small_entry == NULL)
                                return -ENOSPC;
                        entry = small_entry;
                }

                entry_size = EXT4_XATTR_LEN(entry->e_name_len);
                total_size = entry_size;
                if (!entry->e_value_inum)
                        total_size += EXT4_XATTR_SIZE(
                                              le32_to_cpu(entry->e_value_size));
                error = ext4_xattr_move_to_block(handle, inode, raw_inode,
                                                 entry);
                if (error)
                        return error;

                *total_ino -= entry_size;
                ifree += total_size;
                bfree -= total_size;
        }

        return 0;
}

/*
 * Expand an inode by new_extra_isize bytes when EAs are present.
 * Returns 0 on success or negative error number on failure.
 */
int ext4_expand_extra_isize_ea(struct inode *inode, int new_extra_isize,
                               struct ext4_inode *raw_inode, handle_t *handle)
{
        struct ext4_xattr_ibody_header *header;
        struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
        static unsigned int mnt_count;
        size_t min_offs;
        size_t ifree, bfree;
        int total_ino;
        void *base, *end;
        int error = 0, tried_min_extra_isize = 0;
        int s_min_extra_isize = le16_to_cpu(sbi->s_es->s_min_extra_isize);
        int isize_diff;        /* How much do we need to grow i_extra_isize */

retry:
        isize_diff = new_extra_isize - EXT4_I(inode)->i_extra_isize;
        if (EXT4_I(inode)->i_extra_isize >= new_extra_isize)
                return 0;

        header = IHDR(inode, raw_inode);

        /*
         * Check if enough free space is available in the inode to shift the
         * entries ahead by new_extra_isize.
         */

        base = IFIRST(header);
        end = ITAIL(inode, raw_inode);
        min_offs = end - base;
        total_ino = sizeof(struct ext4_xattr_ibody_header) + sizeof(u32);

        ifree = ext4_xattr_free_space(base, &min_offs, base, &total_ino);
        if (ifree >= isize_diff)
                goto shift;

        /*
         * Enough free space isn't available in the inode, check if
         * EA block can hold new_extra_isize bytes.
         */
        if (EXT4_I(inode)->i_file_acl) {
                struct buffer_head *bh;

                bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO);
                if (IS_ERR(bh)) {
                        error = PTR_ERR(bh);
                        goto cleanup;
                }
                error = ext4_xattr_check_block(inode, bh);
                if (error) {
                        brelse(bh);
                        goto cleanup;
                }
                base = BHDR(bh);
                end = bh->b_data + bh->b_size;
                min_offs = end - base;
                bfree = ext4_xattr_free_space(BFIRST(bh), &min_offs, base,
                                              NULL);
                brelse(bh);
                if (bfree + ifree < isize_diff) {
                        if (!tried_min_extra_isize && s_min_extra_isize) {
                                tried_min_extra_isize++;
                                new_extra_isize = s_min_extra_isize;
                                goto retry;
                        }
                        error = -ENOSPC;
                        goto cleanup;
                }
        } else {
                bfree = inode->i_sb->s_blocksize;
        }

        error = ext4_xattr_make_inode_space(handle, inode, raw_inode,
                                            isize_diff, ifree, bfree,
                                            &total_ino);
        if (error) {
                if (error == -ENOSPC && !tried_min_extra_isize &&
                    s_min_extra_isize) {
                        tried_min_extra_isize++;
                        new_extra_isize = s_min_extra_isize;
                        error = 0;
                        goto retry;
                }
                goto cleanup;
        }
shift:
        /* Adjust the offsets and shift the remaining entries ahead */
        ext4_xattr_shift_entries(IFIRST(header), EXT4_I(inode)->i_extra_isize
                        - new_extra_isize, (void *)raw_inode +
                        EXT4_GOOD_OLD_INODE_SIZE + new_extra_isize,
                        (void *)header, total_ino);
        EXT4_I(inode)->i_extra_isize = new_extra_isize;

        if (ext4_has_inline_data(inode))
                error = ext4_find_inline_data_nolock(inode);

cleanup:
        if (error && (mnt_count != le16_to_cpu(sbi->s_es->s_mnt_count))) {
                ext4_warning(inode->i_sb, "Unable to expand inode %llu. Delete some EAs or run e2fsck.",
                             inode->i_ino);
                mnt_count = le16_to_cpu(sbi->s_es->s_mnt_count);
        }
        return error;
}

/*
 * ext4_xattr_delete_inode()
 *
 * Free extended attribute resources associated with this inode. Traverse
 * all entries and decrement reference on any xattr inodes associated with this
 * inode. This is called immediately before an inode is freed. We have exclusive
 * access to the inode. If an orphan inode is deleted it will also release its
 * references on xattr block and xattr inodes.
 */
int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode,
                            int extra_credits)
{
        struct buffer_head *bh = NULL;
        struct ext4_xattr_ibody_header *header;
        struct ext4_iloc iloc = { .bh = NULL };
        struct ext4_xattr_entry *entry;
        struct inode *ea_inode;
        int error;

        error = ext4_journal_ensure_credits(handle, extra_credits,
                        ext4_free_metadata_revoke_credits(inode->i_sb, 1));
        if (error < 0) {
                EXT4_ERROR_INODE(inode, "ensure credits (error %d)", error);
                goto cleanup;
        }

        if (ext4_has_feature_ea_inode(inode->i_sb) &&
            ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {

                error = ext4_get_inode_loc(inode, &iloc);
                if (error) {
                        EXT4_ERROR_INODE(inode, "inode loc (error %d)", error);
                        goto cleanup;
                }

                error = ext4_journal_get_write_access(handle, inode->i_sb,
                                                iloc.bh, EXT4_JTR_NONE);
                if (error) {
                        EXT4_ERROR_INODE(inode, "write access (error %d)",
                                         error);
                        goto cleanup;
                }

                header = IHDR(inode, ext4_raw_inode(&iloc));
                if (header->h_magic == cpu_to_le32(EXT4_XATTR_MAGIC))
                        ext4_xattr_inode_dec_ref_all(handle, inode, iloc.bh,
                                                     IFIRST(header),
                                                     false /* block_csum */,
                                                     extra_credits,
                                                     false /* skip_quota */);
        }

        if (EXT4_I(inode)->i_file_acl) {
                bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO);
                if (IS_ERR(bh)) {
                        error = PTR_ERR(bh);
                        if (error == -EIO) {
                                EXT4_ERROR_INODE_ERR(inode, EIO,
                                                     "block %llu read error",
                                                     EXT4_I(inode)->i_file_acl);
                        }
                        bh = NULL;
                        goto cleanup;
                }
                error = ext4_xattr_check_block(inode, bh);
                if (error)
                        goto cleanup;

                if (ext4_has_feature_ea_inode(inode->i_sb)) {
                        for (entry = BFIRST(bh); !IS_LAST_ENTRY(entry);
                             entry = EXT4_XATTR_NEXT(entry)) {
                                if (!entry->e_value_inum)
                                        continue;
                                error = ext4_xattr_inode_iget(inode,
                                              le32_to_cpu(entry->e_value_inum),
                                              le32_to_cpu(entry->e_hash),
                                              &ea_inode);
                                if (error)
                                        continue;
                                ext4_xattr_inode_free_quota(inode, ea_inode,
                                              le32_to_cpu(entry->e_value_size));
                                ext4_put_ea_inode(ea_inode);
                        }

                }

                ext4_xattr_release_block(handle, inode, bh,
                                         extra_credits);
                /*
                 * Update i_file_acl value in the same transaction that releases
                 * block.
                 */
                EXT4_I(inode)->i_file_acl = 0;
                error = ext4_mark_inode_dirty(handle, inode);
                if (error) {
                        EXT4_ERROR_INODE(inode, "mark inode dirty (error %d)",
                                         error);
                        goto cleanup;
                }
                ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, handle);
        }
        error = 0;
cleanup:
        brelse(iloc.bh);
        brelse(bh);
        return error;
}

/*
 * Worker function for deferred EA inode iput.  Processes all inodes queued
 * on s_ea_inode_to_free in a context free of xattr_sem/jbd2 handle locks.
 */
static void ext4_ea_inode_work(struct work_struct *work)
{
        struct ext4_sb_info *sbi = container_of(to_delayed_work(work),
                                                struct ext4_sb_info,
                                                s_ea_inode_work);
        struct llist_node *node = llist_del_all(&sbi->s_ea_inode_to_free);

        while (node) {
                struct ext4_inode_info *ei = container_of(node,
                                        struct ext4_inode_info, i_ea_iput_node);
                node = node->next;
                iput(&ei->vfs_inode);
        }
}

/*
 * Release a VFS reference on an EA inode.  Must be used instead of iput()
 * in any context where xattr_sem or a jbd2 handle is held.
 *
 * If this is not the last reference, drops it immediately via
 * iput_if_not_last() with no further action needed.
 *
 * If this is the last reference, the inode is linked onto a per-sb
 * llist via i_ea_iput_node (embedded in ext4_inode_info, sharing space
 * with the unused xattr_sem) and a delayed worker performs the final
 * iput() in a clean context.
 *
 * Note: while an inode is on s_ea_inode_to_free, the unconsumed i_count
 * reference (still 1) keeps it in the inode cache, so any concurrent
 * iget() bumps i_count to >= 2 and iput_if_not_last() will succeed.
 * Nobody will add the inode a second time until ext4_ea_inode_work()
 * drops that reference via iput().
 */
void ext4_put_ea_inode(struct inode *inode)
{
        if (!inode)
                return;
        WARN_ON_ONCE(!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL));
        if (iput_if_not_last(inode))
                return;
        llist_add(&EXT4_I(inode)->i_ea_iput_node,
                  &EXT4_SB(inode->i_sb)->s_ea_inode_to_free);
        /*
         * Use a short delay to allow multiple EA inodes to accumulate,
         * reducing workqueue wakeups when several are released together.
         */
        schedule_delayed_work(&EXT4_SB(inode->i_sb)->s_ea_inode_work, 1);
}

void ext4_init_ea_inode_work(struct ext4_sb_info *sbi)
{
        init_llist_head(&sbi->s_ea_inode_to_free);
        INIT_DELAYED_WORK(&sbi->s_ea_inode_work, ext4_ea_inode_work);
}

/*
 * ext4_xattr_block_cache_insert()
 *
 * Create a new entry in the extended attribute block cache, and insert
 * it unless such an entry is already in the cache.
 */
static void
ext4_xattr_block_cache_insert(struct mb_cache *ea_block_cache,
                              struct buffer_head *bh)
{
        struct ext4_xattr_header *header = BHDR(bh);
        __u32 hash = le32_to_cpu(header->h_hash);
        int reusable = le32_to_cpu(header->h_refcount) <
                       EXT4_XATTR_REFCOUNT_MAX;
        int error;

        if (!ea_block_cache)
                return;
        error = mb_cache_entry_create(ea_block_cache, GFP_NOFS, hash,
                                      bh->b_blocknr, reusable);
        if (error) {
                if (error == -EBUSY)
                        ea_bdebug(bh, "already in cache");
        } else
                ea_bdebug(bh, "inserting [%x]", (int)hash);
}

/*
 * ext4_xattr_cmp()
 *
 * Compare two extended attribute blocks for equality.
 *
 * Returns 0 if the blocks are equal, 1 if they differ.
 */
static int
ext4_xattr_cmp(struct ext4_xattr_header *header1,
               struct ext4_xattr_header *header2)
{
        struct ext4_xattr_entry *entry1, *entry2;

        entry1 = ENTRY(header1+1);
        entry2 = ENTRY(header2+1);
        while (!IS_LAST_ENTRY(entry1)) {
                if (IS_LAST_ENTRY(entry2))
                        return 1;
                if (entry1->e_hash != entry2->e_hash ||
                    entry1->e_name_index != entry2->e_name_index ||
                    entry1->e_name_len != entry2->e_name_len ||
                    entry1->e_value_size != entry2->e_value_size ||
                    entry1->e_value_inum != entry2->e_value_inum ||
                    memcmp(entry1->e_name, entry2->e_name, entry1->e_name_len))
                        return 1;
                if (!entry1->e_value_inum &&
                    memcmp((char *)header1 + le16_to_cpu(entry1->e_value_offs),
                           (char *)header2 + le16_to_cpu(entry2->e_value_offs),
                           le32_to_cpu(entry1->e_value_size)))
                        return 1;

                entry1 = EXT4_XATTR_NEXT(entry1);
                entry2 = EXT4_XATTR_NEXT(entry2);
        }
        if (!IS_LAST_ENTRY(entry2))
                return 1;
        return 0;
}

/*
 * ext4_xattr_block_cache_find()
 *
 * Find an identical extended attribute block.
 *
 * Returns a pointer to the block found, or NULL if such a block was not
 * found, or an error pointer if an error occurred while reading ea block.
 */
static struct buffer_head *
ext4_xattr_block_cache_find(struct inode *inode,
                            struct ext4_xattr_header *header,
                            struct mb_cache_entry **pce)
{
        __u32 hash = le32_to_cpu(header->h_hash);
        struct mb_cache_entry *ce;
        struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode);

        if (!ea_block_cache)
                return NULL;
        if (!header->h_hash)
                return NULL;  /* never share */
        ea_idebug(inode, "looking for cached blocks [%x]", (int)hash);
        ce = mb_cache_entry_find_first(ea_block_cache, hash);
        while (ce) {
                struct buffer_head *bh;

                bh = ext4_sb_bread(inode->i_sb, ce->e_value, REQ_PRIO);
                if (IS_ERR(bh)) {
                        if (PTR_ERR(bh) != -ENOMEM)
                                EXT4_ERROR_INODE(inode, "block %lu read error",
                                                 (unsigned long)ce->e_value);
                        mb_cache_entry_put(ea_block_cache, ce);
                        return bh;
                } else if (ext4_xattr_cmp(header, BHDR(bh)) == 0) {
                        *pce = ce;
                        return bh;
                }
                brelse(bh);
                ce = mb_cache_entry_find_next(ea_block_cache, ce);
        }
        return NULL;
}

#define NAME_HASH_SHIFT 5
#define VALUE_HASH_SHIFT 16

/*
 * ext4_xattr_hash_entry()
 *
 * Compute the hash of an extended attribute.
 */
static __le32 ext4_xattr_hash_entry(char *name, size_t name_len, __le32 *value,
                                    size_t value_count)
{
        __u32 hash = 0;

        while (name_len--) {
                hash = (hash << NAME_HASH_SHIFT) ^
                       (hash >> (8*sizeof(hash) - NAME_HASH_SHIFT)) ^
                       (unsigned char)*name++;
        }
        while (value_count--) {
                hash = (hash << VALUE_HASH_SHIFT) ^
                       (hash >> (8*sizeof(hash) - VALUE_HASH_SHIFT)) ^
                       le32_to_cpu(*value++);
        }
        return cpu_to_le32(hash);
}

/*
 * ext4_xattr_hash_entry_signed()
 *
 * Compute the hash of an extended attribute incorrectly.
 */
static __le32 ext4_xattr_hash_entry_signed(char *name, size_t name_len, __le32 *value, size_t value_count)
{
        __u32 hash = 0;

        while (name_len--) {
                hash = (hash << NAME_HASH_SHIFT) ^
                       (hash >> (8*sizeof(hash) - NAME_HASH_SHIFT)) ^
                       (signed char)*name++;
        }
        while (value_count--) {
                hash = (hash << VALUE_HASH_SHIFT) ^
                       (hash >> (8*sizeof(hash) - VALUE_HASH_SHIFT)) ^
                       le32_to_cpu(*value++);
        }
        return cpu_to_le32(hash);
}

#undef NAME_HASH_SHIFT
#undef VALUE_HASH_SHIFT

#define BLOCK_HASH_SHIFT 16

/*
 * ext4_xattr_rehash()
 *
 * Re-compute the extended attribute hash value after an entry has changed.
 */
static void ext4_xattr_rehash(struct ext4_xattr_header *header)
{
        struct ext4_xattr_entry *here;
        __u32 hash = 0;

        here = ENTRY(header+1);
        while (!IS_LAST_ENTRY(here)) {
                if (!here->e_hash) {
                        /* Block is not shared if an entry's hash value == 0 */
                        hash = 0;
                        break;
                }
                hash = (hash << BLOCK_HASH_SHIFT) ^
                       (hash >> (8*sizeof(hash) - BLOCK_HASH_SHIFT)) ^
                       le32_to_cpu(here->e_hash);
                here = EXT4_XATTR_NEXT(here);
        }
        header->h_hash = cpu_to_le32(hash);
}

#undef BLOCK_HASH_SHIFT

#define        HASH_BUCKET_BITS        10

struct mb_cache *
ext4_xattr_create_cache(void)
{
        return mb_cache_create(HASH_BUCKET_BITS);
}

void ext4_xattr_destroy_cache(struct mb_cache *cache)
{
        if (cache)
                mb_cache_destroy(cache);
}
































































































































































































































    8 

























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
 * Written by Alex Tomas <alex@clusterfs.com>
 */

#ifndef _EXT4_EXTENTS
#define _EXT4_EXTENTS

#include "ext4.h"

/*
 * With AGGRESSIVE_TEST defined, the capacity of index/leaf blocks
 * becomes very small, so index split, in-depth growing and
 * other hard changes happen much more often.
 * This is for debug purposes only.
 */
#define AGGRESSIVE_TEST_

/*
 * With EXTENTS_STATS defined, the number of blocks and extents
 * are collected in the truncate path. They'll be shown at
 * umount time.
 */
#define EXTENTS_STATS__

/*
 * If CHECK_BINSEARCH is defined, then the results of the binary search
 * will also be checked by linear search.
 */
#define CHECK_BINSEARCH__

/*
 * ext4_inode has i_block array (60 bytes total).
 * The first 12 bytes store ext4_extent_header;
 * the remainder stores an array of ext4_extent.
 * For non-inode extent blocks, ext4_extent_tail
 * follows the array.
 */

/*
 * This is the extent tail on-disk structure.
 * All other extent structures are 12 bytes long.  It turns out that
 * block_size % 12 >= 4 for at least all powers of 2 greater than 512, which
 * covers all valid ext4 block sizes.  Therefore, this tail structure can be
 * crammed into the end of the block without having to rebalance the tree.
 */
struct ext4_extent_tail {
        __le32        et_checksum;        /* crc32c(uuid+inum+extent_block) */
};

/*
 * This is the extent on-disk structure.
 * It's used at the bottom of the tree.
 */
struct ext4_extent {
        __le32        ee_block;        /* first logical block extent covers */
        __le16        ee_len;                /* number of blocks covered by extent */
        __le16        ee_start_hi;        /* high 16 bits of physical block */
        __le32        ee_start_lo;        /* low 32 bits of physical block */
};

/*
 * This is index on-disk structure.
 * It's used at all the levels except the bottom.
 */
struct ext4_extent_idx {
        __le32        ei_block;        /* index covers logical blocks from 'block' */
        __le32        ei_leaf_lo;        /* pointer to the physical block of the next *
                                 * level. leaf or next index could be there */
        __le16        ei_leaf_hi;        /* high 16 bits of physical block */
        __u16        ei_unused;
};

/*
 * Each block (leaves and indexes), even inode-stored has header.
 */
struct ext4_extent_header {
        __le16        eh_magic;        /* probably will support different formats */
        __le16        eh_entries;        /* number of valid entries */
        __le16        eh_max;                /* capacity of store in entries */
        __le16        eh_depth;        /* has tree real underlying blocks? */
        __le32        eh_generation;        /* generation of the tree */
};

#define EXT4_EXT_MAGIC                cpu_to_le16(0xf30a)
#define EXT4_MAX_EXTENT_DEPTH 5

#define EXT4_EXTENT_TAIL_OFFSET(hdr) \
        (sizeof(struct ext4_extent_header) + \
         (sizeof(struct ext4_extent) * le16_to_cpu((hdr)->eh_max)))

static inline struct ext4_extent_tail *
find_ext4_extent_tail(struct ext4_extent_header *eh)
{
        return (struct ext4_extent_tail *)(((void *)eh) +
                                           EXT4_EXTENT_TAIL_OFFSET(eh));
}

/*
 * Array of ext4_ext_path contains path to some extent.
 * Creation/lookup routines use it for traversal/splitting/etc.
 * Truncate uses it to simulate recursive walking.
 */
struct ext4_ext_path {
        ext4_fsblk_t                        p_block;
        __u16                                p_depth;
        __u16                                p_maxdepth;
        struct ext4_extent                *p_ext;
        struct ext4_extent_idx                *p_idx;
        struct ext4_extent_header        *p_hdr;
        struct buffer_head                *p_bh;
};

/*
 * Used to record a portion of a cluster found at the beginning or end
 * of an extent while traversing the extent tree during space removal.
 * A partial cluster may be removed if it does not contain blocks shared
 * with extents that aren't being deleted (tofree state).  Otherwise,
 * it cannot be removed (nofree state).
 */
struct partial_cluster {
        ext4_fsblk_t pclu;  /* physical cluster number */
        ext4_lblk_t lblk;   /* logical block number within logical cluster */
        enum {initial, tofree, nofree} state;
};

/*
 * structure for external API
 */

/*
 * EXT_INIT_MAX_LEN is the maximum number of blocks we can have in an
 * initialized extent. This is 2^15 and not (2^16 - 1), since we use the
 * MSB of ee_len field in the extent datastructure to signify if this
 * particular extent is an initialized extent or an unwritten (i.e.
 * preallocated).
 * EXT_UNWRITTEN_MAX_LEN is the maximum number of blocks we can have in an
 * unwritten extent.
 * If ee_len is <= 0x8000, it is an initialized extent. Otherwise, it is an
 * unwritten one. In other words, if MSB of ee_len is set, it is an
 * unwritten extent with only one special scenario when ee_len = 0x8000.
 * In this case we can not have an unwritten extent of zero length and
 * thus we make it as a special case of initialized extent with 0x8000 length.
 * This way we get better extent-to-group alignment for initialized extents.
 * Hence, the maximum number of blocks we can have in an *initialized*
 * extent is 2^15 (32768) and in an *unwritten* extent is 2^15-1 (32767).
 */
#define EXT_INIT_MAX_LEN        (1UL << 15)
#define EXT_UNWRITTEN_MAX_LEN        (EXT_INIT_MAX_LEN - 1)


#define EXT_FIRST_EXTENT(__hdr__) \
        ((struct ext4_extent *) (((char *) (__hdr__)) +                \
                                 sizeof(struct ext4_extent_header)))
#define EXT_FIRST_INDEX(__hdr__) \
        ((struct ext4_extent_idx *) (((char *) (__hdr__)) +        \
                                     sizeof(struct ext4_extent_header)))
#define EXT_HAS_FREE_INDEX(__path__) \
        (le16_to_cpu((__path__)->p_hdr->eh_entries) \
                                     < le16_to_cpu((__path__)->p_hdr->eh_max))
#define EXT_LAST_EXTENT(__hdr__) \
        (EXT_FIRST_EXTENT((__hdr__)) + le16_to_cpu((__hdr__)->eh_entries) - 1)
#define EXT_LAST_INDEX(__hdr__) \
        (EXT_FIRST_INDEX((__hdr__)) + le16_to_cpu((__hdr__)->eh_entries) - 1)
#define EXT_MAX_EXTENT(__hdr__)        \
        ((le16_to_cpu((__hdr__)->eh_max)) ? \
        ((EXT_FIRST_EXTENT((__hdr__)) + le16_to_cpu((__hdr__)->eh_max) - 1)) \
                                        : NULL)
#define EXT_MAX_INDEX(__hdr__) \
        ((le16_to_cpu((__hdr__)->eh_max)) ? \
        ((EXT_FIRST_INDEX((__hdr__)) + le16_to_cpu((__hdr__)->eh_max) - 1)) \
                                        : NULL)

static inline struct ext4_extent_header *ext_inode_hdr(struct inode *inode)
{
        return (struct ext4_extent_header *) EXT4_I(inode)->i_data;
}

static inline struct ext4_extent_header *ext_block_hdr(struct buffer_head *bh)
{
        return (struct ext4_extent_header *) bh->b_data;
}

static inline unsigned short ext_depth(struct inode *inode)
{
        return le16_to_cpu(ext_inode_hdr(inode)->eh_depth);
}

static inline void ext4_ext_mark_unwritten(struct ext4_extent *ext)
{
        /* We can not have an unwritten extent of zero length! */
        BUG_ON((le16_to_cpu(ext->ee_len) & ~EXT_INIT_MAX_LEN) == 0);
        ext->ee_len |= cpu_to_le16(EXT_INIT_MAX_LEN);
}

static inline int ext4_ext_is_unwritten(struct ext4_extent *ext)
{
        /* Extent with ee_len of 0x8000 is treated as an initialized extent */
        return (le16_to_cpu(ext->ee_len) > EXT_INIT_MAX_LEN);
}

static inline int ext4_ext_get_actual_len(struct ext4_extent *ext)
{
        return (le16_to_cpu(ext->ee_len) <= EXT_INIT_MAX_LEN ?
                le16_to_cpu(ext->ee_len) :
                (le16_to_cpu(ext->ee_len) - EXT_INIT_MAX_LEN));
}

static inline void ext4_ext_mark_initialized(struct ext4_extent *ext)
{
        ext->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ext));
}

/*
 * ext4_ext_pblock:
 * combine low and high parts of physical block number into ext4_fsblk_t
 */
static inline ext4_fsblk_t ext4_ext_pblock(struct ext4_extent *ex)
{
        ext4_fsblk_t block;

        block = le32_to_cpu(ex->ee_start_lo);
        block |= ((ext4_fsblk_t) le16_to_cpu(ex->ee_start_hi) << 31) << 1;
        return block;
}

/*
 * ext4_idx_pblock:
 * combine low and high parts of a leaf physical block number into ext4_fsblk_t
 */
static inline ext4_fsblk_t ext4_idx_pblock(struct ext4_extent_idx *ix)
{
        ext4_fsblk_t block;

        block = le32_to_cpu(ix->ei_leaf_lo);
        block |= ((ext4_fsblk_t) le16_to_cpu(ix->ei_leaf_hi) << 31) << 1;
        return block;
}

/*
 * ext4_ext_store_pblock:
 * stores a large physical block number into an extent struct,
 * breaking it into parts
 */
static inline void ext4_ext_store_pblock(struct ext4_extent *ex,
                                         ext4_fsblk_t pb)
{
        ex->ee_start_lo = cpu_to_le32((unsigned long) (pb & 0xffffffff));
        ex->ee_start_hi = cpu_to_le16((unsigned long) ((pb >> 31) >> 1) &
                                      0xffff);
}

/*
 * ext4_idx_store_pblock:
 * stores a large physical block number into an index struct,
 * breaking it into parts
 */
static inline void ext4_idx_store_pblock(struct ext4_extent_idx *ix,
                                         ext4_fsblk_t pb)
{
        ix->ei_leaf_lo = cpu_to_le32((unsigned long) (pb & 0xffffffff));
        ix->ei_leaf_hi = cpu_to_le16((unsigned long) ((pb >> 31) >> 1) &
                                     0xffff);
}

extern int __ext4_ext_dirty(const char *where, unsigned int line,
                            handle_t *handle, struct inode *inode,
                            struct ext4_ext_path *path);
extern int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex);
#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS)
extern int ext4_ext_space_root_idx_test(struct inode *inode, int check);
extern struct ext4_ext_path *ext4_split_convert_extents_test(
                                handle_t *handle, struct inode *inode,
                                struct ext4_map_blocks *map,
                                struct ext4_ext_path *path,
                                int flags, unsigned int *allocated);
#endif
#endif /* _EXT4_EXTENTS */



























































    1 
    1 
    1 






    1 










   11 

    9 
   10 



   10 



   11 












    2 





    3 

























    7 








    7 






    1 



























    1 





    1 






    1 



    1 


    1 




    1 




















    1 









    1 

    1 






    1 







    1 


    1 
    1 

    1 


    1 























































































    4 




    1 


    3 






    1 
    1 






    1 
    1 











    7 





    2 









    1 




    5 











   14 








    1 


    2 


   11 




    1 










    2 








    5 


















    7 









    3 















    4 




    1 




















    3 





    3 
    3 







    3 









    3 
























    2 








    2 

















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  Point-to-Point Tunneling Protocol for Linux
 *
 *        Authors: Dmitry Kozlov <xeb@mail.ru>
 */

#include <linux/string.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/netdevice.h>
#include <linux/net.h>
#include <linux/skbuff.h>
#include <linux/vmalloc.h>
#include <linux/init.h>
#include <linux/ppp_channel.h>
#include <linux/ppp_defs.h>
#include <linux/if_pppox.h>
#include <linux/ppp-ioctl.h>
#include <linux/notifier.h>
#include <linux/file.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/rcupdate.h>
#include <linux/security.h>
#include <linux/spinlock.h>

#include <net/sock.h>
#include <net/protocol.h>
#include <net/ip.h>
#include <net/icmp.h>
#include <net/route.h>
#include <net/gre.h>
#include <net/pptp.h>

#include <linux/uaccess.h>

#define PPTP_DRIVER_VERSION "0.8.5"

#define MAX_CALLID 65535

static DECLARE_BITMAP(callid_bitmap, MAX_CALLID + 1);
static struct pppox_sock __rcu **callid_sock;

static DEFINE_SPINLOCK(chan_lock);

static struct proto pptp_sk_proto __read_mostly;
static const struct ppp_channel_ops pptp_chan_ops;
static const struct proto_ops pptp_ops;

static struct pppox_sock *lookup_chan(u16 call_id, __be32 s_addr)
{
        struct pppox_sock *sock;
        struct pptp_opt *opt;

        rcu_read_lock();
        sock = rcu_dereference(callid_sock[call_id]);
        if (sock) {
                opt = &sock->proto.pptp;
                if (opt->dst_addr.sin_addr.s_addr != s_addr)
                        sock = NULL;
                else
                        sock_hold(&sock->sk);
        }
        rcu_read_unlock();

        return sock;
}

static int lookup_chan_dst(u16 call_id, __be32 d_addr)
{
        struct pppox_sock *sock;
        struct pptp_opt *opt;
        int i;

        rcu_read_lock();
        i = 1;
        for_each_set_bit_from(i, callid_bitmap, MAX_CALLID) {
                sock = rcu_dereference(callid_sock[i]);
                if (!sock)
                        continue;
                opt = &sock->proto.pptp;
                if (opt->dst_addr.call_id == call_id &&
                          opt->dst_addr.sin_addr.s_addr == d_addr)
                        break;
        }
        rcu_read_unlock();

        return i < MAX_CALLID;
}

static int add_chan(struct pppox_sock *sock,
                    struct pptp_addr *sa)
{
        static int call_id;

        spin_lock(&chan_lock);
        if (!sa->call_id)        {
                call_id = find_next_zero_bit(callid_bitmap, MAX_CALLID, call_id + 1);
                if (call_id == MAX_CALLID) {
                        call_id = find_next_zero_bit(callid_bitmap, MAX_CALLID, 1);
                        if (call_id == MAX_CALLID)
                                goto out_err;
                }
                sa->call_id = call_id;
        } else if (test_bit(sa->call_id, callid_bitmap)) {
                goto out_err;
        }

        sock->proto.pptp.src_addr = *sa;
        set_bit(sa->call_id, callid_bitmap);
        rcu_assign_pointer(callid_sock[sa->call_id], sock);
        spin_unlock(&chan_lock);

        return 0;

out_err:
        spin_unlock(&chan_lock);
        return -1;
}

static void del_chan(struct pppox_sock *sock)
{
        spin_lock(&chan_lock);
        clear_bit(sock->proto.pptp.src_addr.call_id, callid_bitmap);
        RCU_INIT_POINTER(callid_sock[sock->proto.pptp.src_addr.call_id], NULL);
        spin_unlock(&chan_lock);
}

static struct rtable *pptp_route_output(const struct pppox_sock *po,
                                        struct flowi4 *fl4)
{
        const struct sock *sk = &po->sk;
        struct net *net;

        net = sock_net(sk);
        flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark, 0,
                           RT_SCOPE_UNIVERSE, IPPROTO_GRE, 0,
                           po->proto.pptp.dst_addr.sin_addr.s_addr,
                           po->proto.pptp.src_addr.sin_addr.s_addr,
                           0, 0, sock_net_uid(net, sk));
        security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4));

        return ip_route_output_flow(net, fl4, sk);
}

static int pptp_xmit(struct ppp_channel *chan, struct sk_buff *skb)
{
        struct sock *sk = chan->private;
        struct pppox_sock *po = pppox_sk(sk);
        struct net *net = sock_net(sk);
        struct pptp_opt *opt = &po->proto.pptp;
        struct pptp_gre_header *hdr;
        unsigned int header_len = sizeof(*hdr);
        struct flowi4 fl4;
        int islcp;
        int len;
        unsigned char *data;
        __u32 seq_recv;
        struct rtable *rt;
        struct net_device *tdev;
        struct iphdr  *iph;
        int    max_headroom;

        if (po->sk.sk_state & PPPOX_DEAD)
                goto tx_drop;

        rt = pptp_route_output(po, &fl4);
        if (IS_ERR(rt))
                goto tx_drop;

        tdev = rt->dst.dev;

        max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(*iph) + sizeof(*hdr) + 2;

        if (skb_headroom(skb) < max_headroom || skb_cloned(skb) || skb_shared(skb)) {
                struct sk_buff *new_skb = skb_realloc_headroom(skb, max_headroom);

                if (!new_skb)
                        goto tx_error;

                if (skb->sk)
                        skb_set_owner_w(new_skb, skb->sk);
                consume_skb(skb);
                skb = new_skb;
        }

        /* Ensure we can safely access protocol field and LCP code */
        if (!pskb_may_pull(skb, 3))
                goto tx_error;

        data = skb->data;
        islcp = ((data[0] << 8) + data[1]) == PPP_LCP && 1 <= data[2] && data[2] <= 7;

        /* compress protocol field */
        if ((opt->ppp_flags & SC_COMP_PROT) && data[0] == 0 && !islcp)
                skb_pull(skb, 1);

        /* Put in the address/control bytes if necessary */
        if ((opt->ppp_flags & SC_COMP_AC) == 0 || islcp) {
                data = skb_push(skb, 2);
                data[0] = PPP_ALLSTATIONS;
                data[1] = PPP_UI;
        }

        len = skb->len;

        seq_recv = opt->seq_recv;

        if (opt->ack_sent == seq_recv)
                header_len -= sizeof(hdr->ack);

        /* Push down and install GRE header */
        skb_push(skb, header_len);
        hdr = (struct pptp_gre_header *)(skb->data);

        hdr->gre_hd.flags = GRE_KEY | GRE_VERSION_1 | GRE_SEQ;
        hdr->gre_hd.protocol = GRE_PROTO_PPP;
        hdr->call_id = htons(opt->dst_addr.call_id);

        hdr->seq = htonl(++opt->seq_sent);
        if (opt->ack_sent != seq_recv)        {
                /* send ack with this message */
                hdr->gre_hd.flags |= GRE_ACK;
                hdr->ack  = htonl(seq_recv);
                opt->ack_sent = seq_recv;
        }
        hdr->payload_len = htons(len);

        /*        Push down and install the IP header. */

        skb_reset_transport_header(skb);
        skb_push(skb, sizeof(*iph));
        skb_reset_network_header(skb);
        memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
        IPCB(skb)->flags &= ~(IPSKB_XFRM_TUNNEL_SIZE | IPSKB_XFRM_TRANSFORMED | IPSKB_REROUTED);

        iph =        ip_hdr(skb);
        iph->version =        4;
        iph->ihl =        sizeof(struct iphdr) >> 2;
        if (ip_dont_fragment(sk, &rt->dst))
                iph->frag_off        =        htons(IP_DF);
        else
                iph->frag_off        =        0;
        iph->protocol = IPPROTO_GRE;
        iph->tos      = 0;
        iph->daddr    = fl4.daddr;
        iph->saddr    = fl4.saddr;
        iph->ttl      = ip4_dst_hoplimit(&rt->dst);
        iph->tot_len  = htons(skb->len);

        skb_dst_drop(skb);
        skb_dst_set(skb, &rt->dst);

        nf_reset_ct(skb);

        skb->ip_summed = CHECKSUM_NONE;
        ip_select_ident(net, skb, NULL);
        ip_send_check(iph);

        ip_local_out(net, skb->sk, skb);
        return 1;

tx_error:
        ip_rt_put(rt);
tx_drop:
        kfree_skb(skb);
        return 1;
}

static int pptp_rcv_core(struct sock *sk, struct sk_buff *skb)
{
        struct pppox_sock *po = pppox_sk(sk);
        struct pptp_opt *opt = &po->proto.pptp;
        int headersize, payload_len, seq;
        __u8 *payload;
        struct pptp_gre_header *header;

        if (!(sk->sk_state & PPPOX_CONNECTED))
                goto drop;

        header = (struct pptp_gre_header *)(skb->data);
        headersize  = sizeof(*header);

        /* test if acknowledgement present */
        if (GRE_IS_ACK(header->gre_hd.flags)) {
                __u32 ack;

                if (!pskb_may_pull(skb, headersize))
                        goto drop;
                header = (struct pptp_gre_header *)(skb->data);

                /* ack in different place if S = 0 */
                ack = GRE_IS_SEQ(header->gre_hd.flags) ? ntohl(header->ack) :
                                                         ntohl(header->seq);
                if (ack > opt->ack_recv)
                        opt->ack_recv = ack;
                /* also handle sequence number wrap-around  */
                if (WRAPPED(ack, opt->ack_recv))
                        opt->ack_recv = ack;
        } else {
                headersize -= sizeof(header->ack);
        }
        /* test if payload present */
        if (!GRE_IS_SEQ(header->gre_hd.flags))
                goto drop;

        payload_len = ntohs(header->payload_len);
        seq         = ntohl(header->seq);

        /* check for incomplete packet (length smaller than expected) */
        if (!pskb_may_pull(skb, headersize + payload_len))
                goto drop;

        payload = skb->data + headersize;
        /* check for expected sequence number */
        if (seq < opt->seq_recv + 1 || WRAPPED(opt->seq_recv, seq)) {
                if ((payload[0] == PPP_ALLSTATIONS) && (payload[1] == PPP_UI) &&
                                (PPP_PROTOCOL(payload) == PPP_LCP) &&
                                ((payload[4] == PPP_LCP_ECHOREQ) || (payload[4] == PPP_LCP_ECHOREP)))
                        goto allow_packet;
        } else {
                opt->seq_recv = seq;
allow_packet:
                skb_pull(skb, headersize);

                if (payload[0] == PPP_ALLSTATIONS && payload[1] == PPP_UI) {
                        /* chop off address/control */
                        if (skb->len < 3)
                                goto drop;
                        skb_pull(skb, 2);
                }

                skb->ip_summed = CHECKSUM_NONE;
                skb_set_network_header(skb, skb->head-skb->data);
                ppp_input(&po->chan, skb);

                return NET_RX_SUCCESS;
        }
drop:
        kfree_skb(skb);
        return NET_RX_DROP;
}

static int pptp_rcv(struct sk_buff *skb)
{
        struct pppox_sock *po;
        struct pptp_gre_header *header;
        struct iphdr *iph;

        if (skb->pkt_type != PACKET_HOST)
                goto drop;

        if (!pskb_may_pull(skb, 12))
                goto drop;

        iph = ip_hdr(skb);

        header = (struct pptp_gre_header *)skb->data;

        if (header->gre_hd.protocol != GRE_PROTO_PPP || /* PPTP-GRE protocol for PPTP */
                GRE_IS_CSUM(header->gre_hd.flags) ||    /* flag CSUM should be clear */
                GRE_IS_ROUTING(header->gre_hd.flags) || /* flag ROUTING should be clear */
                !GRE_IS_KEY(header->gre_hd.flags) ||    /* flag KEY should be set */
                (header->gre_hd.flags & GRE_FLAGS))     /* flag Recursion Ctrl should be clear */
                /* if invalid, discard this packet */
                goto drop;

        po = lookup_chan(ntohs(header->call_id), iph->saddr);
        if (po) {
                skb_dst_drop(skb);
                nf_reset_ct(skb);
                return sk_receive_skb(&po->sk, skb, 0);
        }
drop:
        kfree_skb(skb);
        return NET_RX_DROP;
}

static int pptp_bind(struct socket *sock, struct sockaddr_unsized *uservaddr,
                     int sockaddr_len)
{
        struct sock *sk = sock->sk;
        struct sockaddr_pppox *sp = (struct sockaddr_pppox *) uservaddr;
        struct pppox_sock *po = pppox_sk(sk);
        int error = 0;

        if (sockaddr_len < sizeof(struct sockaddr_pppox))
                return -EINVAL;

        lock_sock(sk);

        if (sk->sk_state & PPPOX_DEAD) {
                error = -EALREADY;
                goto out;
        }

        if (sk->sk_state & PPPOX_BOUND) {
                error = -EBUSY;
                goto out;
        }

        if (add_chan(po, &sp->sa_addr.pptp))
                error = -EBUSY;
        else
                sk->sk_state |= PPPOX_BOUND;

out:
        release_sock(sk);
        return error;
}

static int pptp_connect(struct socket *sock, struct sockaddr_unsized *uservaddr,
                        int sockaddr_len, int flags)
{
        struct sock *sk = sock->sk;
        struct sockaddr_pppox *sp = (struct sockaddr_pppox *) uservaddr;
        struct pppox_sock *po = pppox_sk(sk);
        struct pptp_opt *opt = &po->proto.pptp;
        struct rtable *rt;
        struct flowi4 fl4;
        int error = 0;

        if (sockaddr_len < sizeof(struct sockaddr_pppox))
                return -EINVAL;

        if (sp->sa_protocol != PX_PROTO_PPTP)
                return -EINVAL;

        if (lookup_chan_dst(sp->sa_addr.pptp.call_id, sp->sa_addr.pptp.sin_addr.s_addr))
                return -EALREADY;

        lock_sock(sk);
        /* Check for already bound sockets */
        if (sk->sk_state & PPPOX_CONNECTED) {
                error = -EBUSY;
                goto end;
        }

        /* Check for already disconnected sockets, on attempts to disconnect */
        if (sk->sk_state & PPPOX_DEAD) {
                error = -EALREADY;
                goto end;
        }

        if (!opt->src_addr.sin_addr.s_addr || !sp->sa_addr.pptp.sin_addr.s_addr) {
                error = -EINVAL;
                goto end;
        }

        po->chan.private = sk;
        po->chan.ops = &pptp_chan_ops;

        rt = pptp_route_output(po, &fl4);
        if (IS_ERR(rt)) {
                error = -EHOSTUNREACH;
                goto end;
        }
        sk_setup_caps(sk, &rt->dst);

        po->chan.mtu = dst_mtu(&rt->dst);
        if (!po->chan.mtu)
                po->chan.mtu = PPP_MRU;
        po->chan.mtu -= PPTP_HEADER_OVERHEAD;

        po->chan.hdrlen = 2 + sizeof(struct pptp_gre_header);
        po->chan.direct_xmit = true;
        error = ppp_register_channel(&po->chan);
        if (error) {
                pr_err("PPTP: failed to register PPP channel (%d)\n", error);
                goto end;
        }

        opt->dst_addr = sp->sa_addr.pptp;
        sk->sk_state |= PPPOX_CONNECTED;

 end:
        release_sock(sk);
        return error;
}

static int pptp_getname(struct socket *sock, struct sockaddr *uaddr,
        int peer)
{
        int len = sizeof(struct sockaddr_pppox);
        struct sockaddr_pppox sp;

        memset(&sp.sa_addr, 0, sizeof(sp.sa_addr));

        sp.sa_family    = AF_PPPOX;
        sp.sa_protocol  = PX_PROTO_PPTP;
        sp.sa_addr.pptp = pppox_sk(sock->sk)->proto.pptp.src_addr;

        memcpy(uaddr, &sp, len);

        return len;
}

static int pptp_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct pppox_sock *po;
        int error = 0;

        if (!sk)
                return 0;

        lock_sock(sk);

        if (sock_flag(sk, SOCK_DEAD)) {
                release_sock(sk);
                return -EBADF;
        }

        po = pppox_sk(sk);
        del_chan(po);
        synchronize_rcu();

        pppox_unbind_sock(sk);
        sk->sk_state = PPPOX_DEAD;

        sock_orphan(sk);
        sock->sk = NULL;

        release_sock(sk);
        sock_put(sk);

        return error;
}

static void pptp_sock_destruct(struct sock *sk)
{
        if (!(sk->sk_state & PPPOX_DEAD)) {
                del_chan(pppox_sk(sk));
                pppox_unbind_sock(sk);
        }
        dst_release(rcu_dereference_protected(sk->sk_dst_cache, 1));
}

static int pptp_create(struct net *net, struct socket *sock, int kern)
{
        int error = -ENOMEM;
        struct sock *sk;
        struct pppox_sock *po;
        struct pptp_opt *opt;

        sk = sk_alloc(net, PF_PPPOX, GFP_KERNEL, &pptp_sk_proto, kern);
        if (!sk)
                goto out;

        sock_init_data(sock, sk);

        sock->state = SS_UNCONNECTED;
        sock->ops   = &pptp_ops;

        sk->sk_backlog_rcv = pptp_rcv_core;
        sk->sk_state       = PPPOX_NONE;
        sk->sk_type        = SOCK_STREAM;
        sk->sk_family      = PF_PPPOX;
        sk->sk_protocol    = PX_PROTO_PPTP;
        sk->sk_destruct    = pptp_sock_destruct;

        po = pppox_sk(sk);
        opt = &po->proto.pptp;

        opt->seq_sent = 0; opt->seq_recv = 0xffffffff;
        opt->ack_recv = 0; opt->ack_sent = 0xffffffff;

        error = 0;
out:
        return error;
}

static int pptp_ppp_ioctl(struct ppp_channel *chan, unsigned int cmd,
        unsigned long arg)
{
        struct sock *sk = chan->private;
        struct pppox_sock *po = pppox_sk(sk);
        struct pptp_opt *opt = &po->proto.pptp;
        void __user *argp = (void __user *)arg;
        int __user *p = argp;
        int err, val;

        err = -EFAULT;
        switch (cmd) {
        case PPPIOCGFLAGS:
                val = opt->ppp_flags;
                if (put_user(val, p))
                        break;
                err = 0;
                break;
        case PPPIOCSFLAGS:
                if (get_user(val, p))
                        break;
                opt->ppp_flags = val & ~SC_RCV_BITS;
                err = 0;
                break;
        default:
                err = -ENOTTY;
        }

        return err;
}

static const struct ppp_channel_ops pptp_chan_ops = {
        .start_xmit = pptp_xmit,
        .ioctl      = pptp_ppp_ioctl,
};

static struct proto pptp_sk_proto __read_mostly = {
        .name     = "PPTP",
        .owner    = THIS_MODULE,
        .obj_size = sizeof(struct pppox_sock),
};

static const struct proto_ops pptp_ops = {
        .family     = AF_PPPOX,
        .owner      = THIS_MODULE,
        .release    = pptp_release,
        .bind       = pptp_bind,
        .connect    = pptp_connect,
        .socketpair = sock_no_socketpair,
        .accept     = sock_no_accept,
        .getname    = pptp_getname,
        .listen     = sock_no_listen,
        .shutdown   = sock_no_shutdown,
        .sendmsg    = sock_no_sendmsg,
        .recvmsg    = sock_no_recvmsg,
        .mmap       = sock_no_mmap,
        .ioctl      = pppox_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl = pppox_compat_ioctl,
#endif
};

static const struct pppox_proto pppox_pptp_proto = {
        .create = pptp_create,
        .owner  = THIS_MODULE,
};

static const struct gre_protocol gre_pptp_protocol = {
        .handler = pptp_rcv,
};

static int __init pptp_init_module(void)
{
        int err = 0;
        pr_info("PPTP driver version " PPTP_DRIVER_VERSION "\n");

        callid_sock = vzalloc(array_size(sizeof(void *), (MAX_CALLID + 1)));
        if (!callid_sock)
                return -ENOMEM;

        err = gre_add_protocol(&gre_pptp_protocol, GREPROTO_PPTP);
        if (err) {
                pr_err("PPTP: can't add gre protocol\n");
                goto out_mem_free;
        }

        err = proto_register(&pptp_sk_proto, 0);
        if (err) {
                pr_err("PPTP: can't register sk_proto\n");
                goto out_gre_del_protocol;
        }

        err = register_pppox_proto(PX_PROTO_PPTP, &pppox_pptp_proto);
        if (err) {
                pr_err("PPTP: can't register pppox_proto\n");
                goto out_unregister_sk_proto;
        }

        return 0;

out_unregister_sk_proto:
        proto_unregister(&pptp_sk_proto);
out_gre_del_protocol:
        gre_del_protocol(&gre_pptp_protocol, GREPROTO_PPTP);
out_mem_free:
        vfree(callid_sock);

        return err;
}

static void __exit pptp_exit_module(void)
{
        unregister_pppox_proto(PX_PROTO_PPTP);
        proto_unregister(&pptp_sk_proto);
        gre_del_protocol(&gre_pptp_protocol, GREPROTO_PPTP);
        vfree(callid_sock);
}

module_init(pptp_init_module);
module_exit(pptp_exit_module);

MODULE_DESCRIPTION("Point-to-Point Tunneling Protocol");
MODULE_AUTHOR("D. Kozlov <xeb@mail.ru>");
MODULE_LICENSE("GPL");
MODULE_ALIAS_NET_PF_PROTO(PF_PPPOX, PX_PROTO_PPTP);



















































































































































































































    9 
























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright 2019 Google LLC
 */

#ifndef __LINUX_BLK_CRYPTO_H
#define __LINUX_BLK_CRYPTO_H

#include <linux/minmax.h>
#include <linux/types.h>
#include <uapi/linux/blk-crypto.h>

enum blk_crypto_mode_num {
        BLK_ENCRYPTION_MODE_INVALID,
        BLK_ENCRYPTION_MODE_AES_256_XTS,
        BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV,
        BLK_ENCRYPTION_MODE_ADIANTUM,
        BLK_ENCRYPTION_MODE_SM4_XTS,
        BLK_ENCRYPTION_MODE_MAX,
};

/*
 * Supported types of keys.  Must be bitflags due to their use in
 * blk_crypto_profile::key_types_supported.
 */
enum blk_crypto_key_type {
        /*
         * Raw keys (i.e. "software keys").  These keys are simply kept in raw,
         * plaintext form in kernel memory.
         */
        BLK_CRYPTO_KEY_TYPE_RAW = 0x1,

        /*
         * Hardware-wrapped keys.  These keys are only present in kernel memory
         * in ephemerally-wrapped form, and they can only be unwrapped by
         * dedicated hardware.  For details, see the "Hardware-wrapped keys"
         * section of Documentation/block/inline-encryption.rst.
         */
        BLK_CRYPTO_KEY_TYPE_HW_WRAPPED = 0x2,
};

/*
 * Currently the maximum raw key size is 64 bytes, as that is the key size of
 * BLK_ENCRYPTION_MODE_AES_256_XTS which takes the longest key.
 *
 * The maximum hardware-wrapped key size depends on the hardware's key wrapping
 * algorithm, which is a hardware implementation detail, so it isn't precisely
 * specified.  But currently 128 bytes is plenty in practice.  Implementations
 * are recommended to wrap a 32-byte key for the hardware KDF with AES-256-GCM,
 * which should result in a size closer to 64 bytes than 128.
 *
 * Both of these values can trivially be increased if ever needed.
 */
#define BLK_CRYPTO_MAX_RAW_KEY_SIZE                64
#define BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE        128

#define BLK_CRYPTO_MAX_ANY_KEY_SIZE \
        MAX(BLK_CRYPTO_MAX_RAW_KEY_SIZE, BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE)

/*
 * Size of the "software secret" which can be derived from a hardware-wrapped
 * key.  This is currently always 32 bytes.  Note, the choice of 32 bytes
 * assumes that the software secret is only used directly for algorithms that
 * don't require more than a 256-bit key to get the desired security strength.
 * If it were to be used e.g. directly as an AES-256-XTS key, then this would
 * need to be increased (which is possible if hardware supports it, but care
 * would need to be taken to avoid breaking users who need exactly 32 bytes).
 */
#define BLK_CRYPTO_SW_SECRET_SIZE        32

/* Flags for blk_crypto_config::flags: */

/*
 * If set, inline encryption hardware will be used if available.
 * If unset, CPU-based encryption will always be used (requires
 * CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK)
 */
#define BLK_CRYPTO_CFG_ALLOW_HW                (1 << 0)

/**
 * struct blk_crypto_config - an inline encryption key's crypto configuration
 * @crypto_mode: encryption algorithm this key is for
 * @data_unit_size: the data unit size for all encryption/decryptions with this
 *        key.  This is the size in bytes of each individual plaintext and
 *        ciphertext.  This is always a power of 2.  It might be e.g. the
 *        filesystem block size or the disk sector size.
 * @dun_bytes: the maximum number of bytes of DUN used when using this key
 * @key_type: the type of this key -- either raw or hardware-wrapped
 * @flags: BLK_CRYPTO_CFG_* flags
 */
struct blk_crypto_config {
        enum blk_crypto_mode_num crypto_mode;
        unsigned int data_unit_size;
        unsigned int dun_bytes;
        enum blk_crypto_key_type key_type;
        int flags;
};

/**
 * struct blk_crypto_key - an inline encryption key
 * @crypto_cfg: the crypto mode, data unit size, key type, and other
 *                characteristics of this key and how it will be used
 * @data_unit_size_bits: log2 of data_unit_size
 * @size: size of this key in bytes.  The size of a raw key is fixed for a given
 *          crypto mode, but the size of a hardware-wrapped key can vary.
 * @bytes: the bytes of this key.  Only the first @size bytes are significant.
 *
 * A blk_crypto_key is immutable once created, and many bios can reference it at
 * the same time.  It must not be freed until all bios using it have completed
 * and it has been evicted from all devices on which it may have been used.
 */
struct blk_crypto_key {
        struct blk_crypto_config crypto_cfg;
        unsigned int data_unit_size_bits;
        unsigned int size;
        u8 bytes[BLK_CRYPTO_MAX_ANY_KEY_SIZE];
};

#define BLK_CRYPTO_MAX_IV_SIZE                32
#define BLK_CRYPTO_DUN_ARRAY_SIZE        (BLK_CRYPTO_MAX_IV_SIZE / sizeof(u64))

/**
 * struct bio_crypt_ctx - an inline encryption context
 * @bc_key: the key, algorithm, and data unit size to use
 * @bc_dun: the data unit number (starting IV) to use
 *
 * A bio_crypt_ctx specifies that the contents of the bio will be encrypted (for
 * write requests) or decrypted (for read requests) inline by the storage device
 * or controller, or by the crypto API fallback.
 */
struct bio_crypt_ctx {
        const struct blk_crypto_key        *bc_key;
        u64                                bc_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
};

#include <linux/blk_types.h>
#include <linux/blkdev.h>

#ifdef CONFIG_BLK_INLINE_ENCRYPTION

static inline bool bio_has_crypt_ctx(struct bio *bio)
{
        return bio->bi_crypt_context;
}

static inline struct bio_crypt_ctx *bio_crypt_ctx(struct bio *bio)
{
        return bio->bi_crypt_context;
}

void bio_crypt_set_ctx(struct bio *bio, const struct blk_crypto_key *key,
                       const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],
                       gfp_t gfp_mask);

bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc,
                                 unsigned int bytes,
                                 const u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]);

int blk_crypto_init_key(struct blk_crypto_key *blk_key,
                        const u8 *key_bytes, size_t key_size,
                        enum blk_crypto_key_type key_type,
                        enum blk_crypto_mode_num crypto_mode,
                        unsigned int dun_bytes,
                        unsigned int data_unit_size, int flags);

int blk_crypto_start_using_key(struct block_device *bdev,
                               const struct blk_crypto_key *key);

void blk_crypto_evict_key(struct block_device *bdev,
                          const struct blk_crypto_key *key);

bool blk_crypto_config_supported_natively(struct block_device *bdev,
                                          const struct blk_crypto_config *cfg);

int blk_crypto_derive_sw_secret(struct block_device *bdev,
                                const u8 *eph_key, size_t eph_key_size,
                                u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]);

#else /* CONFIG_BLK_INLINE_ENCRYPTION */

static inline bool bio_has_crypt_ctx(struct bio *bio)
{
        return false;
}

static inline struct bio_crypt_ctx *bio_crypt_ctx(struct bio *bio)
{
        return NULL;
}

#endif /* CONFIG_BLK_INLINE_ENCRYPTION */

bool __blk_crypto_submit_bio(struct bio *bio);

/**
 * blk_crypto_submit_bio - Submit a bio that may have a crypto context
 * @bio: bio to submit
 *
 * If @bio has no crypto context, or the crypt context attached to @bio is
 * supported by the underlying device's inline encryption hardware, just submit
 * @bio.
 *
 * Otherwise, try to perform en/decryption for this bio by falling back to the
 * kernel crypto API. For encryption this means submitting newly allocated
 * bios for the encrypted payload while keeping back the source bio until they
 * complete, while for reads the decryption happens in-place by a hooked in
 * completion handler.
 */
static inline void blk_crypto_submit_bio(struct bio *bio)
{
        if (!bio_has_crypt_ctx(bio) || __blk_crypto_submit_bio(bio))
                submit_bio(bio);
}

int __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask);
/**
 * bio_crypt_clone - clone bio encryption context
 * @dst: destination bio
 * @src: source bio
 * @gfp_mask: memory allocation flags
 *
 * If @src has an encryption context, clone it to @dst.
 *
 * Return: 0 on success, -ENOMEM if out of memory.  -ENOMEM is only possible if
 *           @gfp_mask doesn't include %__GFP_DIRECT_RECLAIM.
 */
static inline int bio_crypt_clone(struct bio *dst, struct bio *src,
                                  gfp_t gfp_mask)
{
        if (bio_has_crypt_ctx(src))
                return __bio_crypt_clone(dst, src, gfp_mask);
        return 0;
}

#endif /* __LINUX_BLK_CRYPTO_H */












































































































































































































































































































































































































    5 













    6 


    5 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
#ifndef _NF_FLOW_TABLE_H
#define _NF_FLOW_TABLE_H

#include <linux/in.h>
#include <linux/in6.h>
#include <linux/netdevice.h>
#include <linux/rhashtable-types.h>
#include <linux/rcupdate.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_conntrack_tuple_common.h>
#include <net/flow_offload.h>
#include <net/dst.h>
#include <linux/if_pppox.h>
#include <linux/ppp_defs.h>

struct nf_flowtable;
struct nf_flow_rule;
struct flow_offload;
enum flow_offload_tuple_dir;

struct nf_flow_key {
        struct flow_dissector_key_meta                        meta;
        struct flow_dissector_key_control                control;
        struct flow_dissector_key_control                enc_control;
        struct flow_dissector_key_basic                        basic;
        struct flow_dissector_key_vlan                        vlan;
        struct flow_dissector_key_vlan                        cvlan;
        union {
                struct flow_dissector_key_ipv4_addrs        ipv4;
                struct flow_dissector_key_ipv6_addrs        ipv6;
        };
        struct flow_dissector_key_keyid                        enc_key_id;
        union {
                struct flow_dissector_key_ipv4_addrs        enc_ipv4;
                struct flow_dissector_key_ipv6_addrs        enc_ipv6;
        };
        struct flow_dissector_key_tcp                        tcp;
        struct flow_dissector_key_ports                        tp;
} __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */

struct nf_flow_match {
        struct flow_dissector        dissector;
        struct nf_flow_key        key;
        struct nf_flow_key        mask;
};

struct nf_flow_rule {
        struct nf_flow_match        match;
        struct flow_rule        *rule;
};

struct nf_flowtable_type {
        struct list_head                list;
        int                                family;
        int                                (*init)(struct nf_flowtable *ft);
        bool                                (*gc)(const struct flow_offload *flow);
        int                                (*setup)(struct nf_flowtable *ft,
                                                 struct net_device *dev,
                                                 enum flow_block_command cmd);
        int                                (*action)(struct net *net,
                                                  struct flow_offload *flow,
                                                  enum flow_offload_tuple_dir dir,
                                                  struct nf_flow_rule *flow_rule);
        void                                (*free)(struct nf_flowtable *ft);
        void                                (*get)(struct nf_flowtable *ft);
        void                                (*put)(struct nf_flowtable *ft);
        nf_hookfn                        *hook;
        struct module                        *owner;
};

enum nf_flowtable_flags {
        NF_FLOWTABLE_HW_OFFLOAD                = 0x1,        /* NFT_FLOWTABLE_HW_OFFLOAD */
        NF_FLOWTABLE_COUNTER                = 0x2,        /* NFT_FLOWTABLE_COUNTER */
};

struct nf_flowtable {
        unsigned int                        flags;                /* readonly in datapath */
        int                                priority;        /* control path (padding hole) */
        struct rhashtable                rhashtable;        /* datapath, read-mostly members come first */

        struct list_head                list;                /* slowpath parts */
        const struct nf_flowtable_type        *type;
        struct delayed_work                gc_work;
        struct flow_block                flow_block;
        struct rw_semaphore                flow_block_lock; /* Guards flow_block */
        possible_net_t                        net;
};

static inline bool nf_flowtable_hw_offload(struct nf_flowtable *flowtable)
{
        return flowtable->flags & NF_FLOWTABLE_HW_OFFLOAD;
}

enum flow_offload_tuple_dir {
        FLOW_OFFLOAD_DIR_ORIGINAL = IP_CT_DIR_ORIGINAL,
        FLOW_OFFLOAD_DIR_REPLY = IP_CT_DIR_REPLY,
};
#define FLOW_OFFLOAD_DIR_MAX        IP_CT_DIR_MAX

enum flow_offload_xmit_type {
        FLOW_OFFLOAD_XMIT_UNSPEC        = 0,
        FLOW_OFFLOAD_XMIT_NEIGH,
        FLOW_OFFLOAD_XMIT_XFRM,
        FLOW_OFFLOAD_XMIT_DIRECT,
        FLOW_OFFLOAD_XMIT_TC,
};

#define NF_FLOW_TABLE_ENCAP_MAX                2

struct flow_offload_tunnel {
        union {
                struct in_addr        src_v4;
                struct in6_addr        src_v6;
        };
        union {
                struct in_addr        dst_v4;
                struct in6_addr        dst_v6;
        };

        u8        inner_proto;
};

struct flow_offload_tuple {
        union {
                struct in_addr                src_v4;
                struct in6_addr                src_v6;
        };
        union {
                struct in_addr                dst_v4;
                struct in6_addr                dst_v6;
        };
        struct {
                __be16                        src_port;
                __be16                        dst_port;
        };

        int                                iifidx;

        u8                                l3proto;
        u8                                l4proto;
        struct {
                u16                        id;
                __be16                        proto;
        } encap[NF_FLOW_TABLE_ENCAP_MAX];

        struct flow_offload_tunnel        tun;

        /* All members above are keys for lookups, see flow_offload_hash(). */
        struct { }                        __hash;

        u16                                dir:2,
                                        xmit_type:3,
                                        encap_num:2,
                                        needs_gso_segment:1,
                                        tun_num:2,
                                        in_vlan_ingress:2;
        u16                                mtu;
        u32                                dst_cookie;
        struct dst_entry                *dst_cache;

        union {
                struct {
                        u32                ifidx;
                };
                struct {
                        u32                ifidx;
                        u8                h_source[ETH_ALEN];
                        u8                h_dest[ETH_ALEN];
                } out;
                struct {
                        u32                iifidx;
                } tc;
        };
};

struct flow_offload_tuple_rhash {
        struct rhash_head                node;
        struct flow_offload_tuple        tuple;
};

enum nf_flow_flags {
        NF_FLOW_SNAT,
        NF_FLOW_DNAT,
        NF_FLOW_CLOSING,
        NF_FLOW_TEARDOWN,
        NF_FLOW_HW,
        NF_FLOW_HW_DYING,
        NF_FLOW_HW_DEAD,
        NF_FLOW_HW_PENDING,
        NF_FLOW_HW_BIDIRECTIONAL,
        NF_FLOW_HW_ESTABLISHED,
};

enum flow_offload_type {
        NF_FLOW_OFFLOAD_UNSPEC        = 0,
        NF_FLOW_OFFLOAD_ROUTE,
};

struct flow_offload {
        struct flow_offload_tuple_rhash                tuplehash[FLOW_OFFLOAD_DIR_MAX];
        struct nf_conn                                *ct;
        unsigned long                                flags;
        u16                                        type;
        u32                                        timeout;
        struct rcu_head                                rcu_head;
};

#define NF_FLOW_TIMEOUT (30 * HZ)
#define nf_flowtable_time_stamp        (u32)jiffies

unsigned long flow_offload_get_timeout(struct flow_offload *flow);

static inline __s32 nf_flow_timeout_delta(unsigned int timeout)
{
        return (__s32)(timeout - nf_flowtable_time_stamp);
}

struct nf_flow_route {
        struct {
                struct dst_entry                *dst;
                struct {
                        u32                        ifindex;
                        struct {
                                u16                id;
                                __be16                proto;
                        } encap[NF_FLOW_TABLE_ENCAP_MAX];
                        struct flow_offload_tunnel tun;
                        u8                        num_encaps:2,
                                                num_tuns:2,
                                                ingress_vlans:2;
                } in;
                struct {
                        u32                        ifindex;
                        u32                        hw_ifindex;
                        u8                        h_source[ETH_ALEN];
                        u8                        h_dest[ETH_ALEN];
                        u8                        needs_gso_segment:1;
                } out;
                enum flow_offload_xmit_type        xmit_type;
        } tuple[FLOW_OFFLOAD_DIR_MAX];
};

struct flow_offload *flow_offload_alloc(struct nf_conn *ct);
void flow_offload_free(struct flow_offload *flow);

struct nft_flowtable;
struct nft_pktinfo;
int nft_flow_route(const struct nft_pktinfo *pkt, const struct nf_conn *ct,
                   struct nf_flow_route *route, enum ip_conntrack_dir dir,
                   struct nft_flowtable *ft);

static inline int
nf_flow_table_offload_add_cb(struct nf_flowtable *flow_table,
                             flow_setup_cb_t *cb, void *cb_priv)
{
        struct flow_block *block = &flow_table->flow_block;
        struct flow_block_cb *block_cb;
        int err = 0;

        down_write(&flow_table->flow_block_lock);
        block_cb = flow_block_cb_lookup(block, cb, cb_priv);
        if (block_cb) {
                err = -EEXIST;
                goto unlock;
        }

        block_cb = flow_block_cb_alloc(cb, cb_priv, cb_priv, NULL);
        if (IS_ERR(block_cb)) {
                err = PTR_ERR(block_cb);
                goto unlock;
        }

        list_add_tail(&block_cb->list, &block->cb_list);
        up_write(&flow_table->flow_block_lock);

        if (flow_table->type->get)
                flow_table->type->get(flow_table);
        return 0;

unlock:
        up_write(&flow_table->flow_block_lock);
        return err;
}

static inline void
nf_flow_table_offload_del_cb(struct nf_flowtable *flow_table,
                             flow_setup_cb_t *cb, void *cb_priv)
{
        struct flow_block *block = &flow_table->flow_block;
        struct flow_block_cb *block_cb;

        down_write(&flow_table->flow_block_lock);
        block_cb = flow_block_cb_lookup(block, cb, cb_priv);
        if (block_cb) {
                list_del(&block_cb->list);
                flow_block_cb_free(block_cb);
        } else {
                WARN_ON(true);
        }
        up_write(&flow_table->flow_block_lock);

        if (flow_table->type->put)
                flow_table->type->put(flow_table);
}

void flow_offload_route_init(struct flow_offload *flow,
                             struct nf_flow_route *route);

int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow);
void flow_offload_refresh(struct nf_flowtable *flow_table,
                          struct flow_offload *flow, bool force);

static inline bool nf_flow_dst_check(struct flow_offload_tuple *tuple)
{
        if (!tuple->dst_cache)
                return true;

        return dst_check(tuple->dst_cache, tuple->dst_cookie);
}

struct flow_offload_tuple_rhash *flow_offload_lookup(struct nf_flowtable *flow_table,
                                                     struct flow_offload_tuple *tuple);
void nf_flow_table_gc_run(struct nf_flowtable *flow_table);
void nf_flow_table_gc_cleanup(struct nf_flowtable *flowtable,
                              struct net_device *dev);
void nf_flow_table_cleanup(struct net_device *dev);

int nf_flow_table_init(struct nf_flowtable *flow_table);
void nf_flow_table_free(struct nf_flowtable *flow_table);

void flow_offload_teardown(struct flow_offload *flow);

void nf_flow_snat_port(const struct flow_offload *flow,
                       struct sk_buff *skb, unsigned int thoff,
                       u8 protocol, enum flow_offload_tuple_dir dir);
void nf_flow_dnat_port(const struct flow_offload *flow,
                       struct sk_buff *skb, unsigned int thoff,
                       u8 protocol, enum flow_offload_tuple_dir dir);

struct flow_ports {
        __be16 source, dest;
};

struct nf_flowtable *nf_flowtable_by_dev(const struct net_device *dev);
int nf_flow_offload_xdp_setup(struct nf_flowtable *flowtable,
                              struct net_device *dev,
                              enum flow_block_command cmd);

unsigned int nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
                                     const struct nf_hook_state *state);
unsigned int nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb,
                                       const struct nf_hook_state *state);

#if (IS_BUILTIN(CONFIG_NF_FLOW_TABLE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \
    (IS_MODULE(CONFIG_NF_FLOW_TABLE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES))
extern int nf_flow_register_bpf(void);
#else
static inline int nf_flow_register_bpf(void)
{
        return 0;
}
#endif

#define MODULE_ALIAS_NF_FLOWTABLE(family)        \
        MODULE_ALIAS("nf-flowtable-" __stringify(family))

void nf_flow_offload_add(struct nf_flowtable *flowtable,
                         struct flow_offload *flow);
void nf_flow_offload_refresh(struct nf_flowtable *flowtable,
                             struct flow_offload *flow);
void nf_flow_offload_del(struct nf_flowtable *flowtable,
                         struct flow_offload *flow);
void nf_flow_offload_stats(struct nf_flowtable *flowtable,
                           struct flow_offload *flow);

void nf_flow_table_offload_flush(struct nf_flowtable *flowtable);
void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable);

int nf_flow_table_offload_setup(struct nf_flowtable *flowtable,
                                struct net_device *dev,
                                enum flow_block_command cmd);
int nf_flow_rule_route_ipv4(struct net *net, struct flow_offload *flow,
                            enum flow_offload_tuple_dir dir,
                            struct nf_flow_rule *flow_rule);
int nf_flow_rule_route_ipv6(struct net *net, struct flow_offload *flow,
                            enum flow_offload_tuple_dir dir,
                            struct nf_flow_rule *flow_rule);

int nf_flow_table_offload_init(void);
void nf_flow_table_offload_exit(void);

static inline __be16 __nf_flow_pppoe_proto(const struct sk_buff *skb)
{
        __be16 proto;

        proto = *((__be16 *)(skb_mac_header(skb) + ETH_HLEN +
                             sizeof(struct pppoe_hdr)));
        switch (proto) {
        case htons(PPP_IP):
                return htons(ETH_P_IP);
        case htons(PPP_IPV6):
                return htons(ETH_P_IPV6);
        }

        return 0;
}

static inline bool nf_flow_pppoe_proto(struct sk_buff *skb, __be16 *inner_proto)
{
        if (!pskb_may_pull(skb, ETH_HLEN + PPPOE_SES_HLEN))
                return false;

        *inner_proto = __nf_flow_pppoe_proto(skb);

        return true;
}

#define NF_FLOW_TABLE_STAT_INC(net, count) __this_cpu_inc((net)->ft.stat->count)
#define NF_FLOW_TABLE_STAT_DEC(net, count) __this_cpu_dec((net)->ft.stat->count)
#define NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count)        \
        this_cpu_inc((net)->ft.stat->count)
#define NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count)        \
        this_cpu_dec((net)->ft.stat->count)

#ifdef CONFIG_NF_FLOW_TABLE_PROCFS
int nf_flow_table_init_proc(struct net *net);
void nf_flow_table_fini_proc(struct net *net);
#else
static inline int nf_flow_table_init_proc(struct net *net)
{
        return 0;
}

static inline void nf_flow_table_fini_proc(struct net *net)
{
}
#endif /* CONFIG_NF_FLOW_TABLE_PROCFS */

#endif /* _NF_FLOW_TABLE_H */

























































































































   29 



























   29 





































































   29 



















































   29 



















   29 
















































































   29 






   29 


























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
// SPDX-License-Identifier: GPL-2.0-or-later
/* RSA asymmetric public-key algorithm [RFC3447]
 *
 * Copyright (c) 2015, Intel Corporation
 * Authors: Tadeusz Struk <tadeusz.struk@intel.com>
 */

#include <linux/fips.h>
#include <linux/module.h>
#include <linux/mpi.h>
#include <crypto/internal/rsa.h>
#include <crypto/internal/akcipher.h>
#include <crypto/akcipher.h>
#include <crypto/algapi.h>

struct rsa_mpi_key {
        MPI n;
        MPI e;
        MPI d;
        MPI p;
        MPI q;
        MPI dp;
        MPI dq;
        MPI qinv;
};

static int rsa_check_payload(MPI x, MPI n)
{
        MPI n1;

        if (mpi_cmp_ui(x, 1) <= 0)
                return -EINVAL;

        n1 = mpi_alloc(0);
        if (!n1)
                return -ENOMEM;

        if (mpi_sub_ui(n1, n, 1) || mpi_cmp(x, n1) >= 0) {
                mpi_free(n1);
                return -EINVAL;
        }

        mpi_free(n1);
        return 0;
}

/*
 * RSAEP function [RFC3447 sec 5.1.1]
 * c = m^e mod n;
 */
static int _rsa_enc(const struct rsa_mpi_key *key, MPI c, MPI m)
{
        /*
         * Even though (1) in RFC3447 only requires 0 <= m <= n - 1, we are
         * slightly more conservative and require 1 < m < n - 1. This is in line
         * with SP 800-56Br2, Section 7.1.1.
         */
        if (rsa_check_payload(m, key->n))
                return -EINVAL;

        /* (2) c = m^e mod n */
        return mpi_powm(c, m, key->e, key->n);
}

/*
 * RSADP function [RFC3447 sec 5.1.2]
 * m_1 = c^dP mod p;
 * m_2 = c^dQ mod q;
 * h = (m_1 - m_2) * qInv mod p;
 * m = m_2 + q * h;
 */
static int _rsa_dec_crt(const struct rsa_mpi_key *key, MPI m_or_m1_or_h, MPI c)
{
        MPI m2, m12_or_qh;
        int ret = -ENOMEM;

        /*
         * Even though (1) in RFC3447 only requires 0 <= c <= n - 1, we are
         * slightly more conservative and require 1 < c < n - 1. This is in line
         * with SP 800-56Br2, Section 7.1.2.
         */
        if (rsa_check_payload(c, key->n))
                return -EINVAL;

        m2 = mpi_alloc(0);
        m12_or_qh = mpi_alloc(0);
        if (!m2 || !m12_or_qh)
                goto err_free_mpi;

        /* (2i) m_1 = c^dP mod p */
        ret = mpi_powm(m_or_m1_or_h, c, key->dp, key->p);
        if (ret)
                goto err_free_mpi;

        /* (2i) m_2 = c^dQ mod q */
        ret = mpi_powm(m2, c, key->dq, key->q);
        if (ret)
                goto err_free_mpi;

        /* (2iii) h = (m_1 - m_2) * qInv mod p */
        ret = mpi_sub(m12_or_qh, m_or_m1_or_h, m2) ?:
              mpi_mulm(m_or_m1_or_h, m12_or_qh, key->qinv, key->p);

        /* (2iv) m = m_2 + q * h */
        ret = ret ?:
              mpi_mul(m12_or_qh, key->q, m_or_m1_or_h) ?:
              mpi_addm(m_or_m1_or_h, m2, m12_or_qh, key->n);

err_free_mpi:
        mpi_free(m12_or_qh);
        mpi_free(m2);
        return ret;
}

static inline struct rsa_mpi_key *rsa_get_key(struct crypto_akcipher *tfm)
{
        return akcipher_tfm_ctx(tfm);
}

static int rsa_enc(struct akcipher_request *req)
{
        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
        const struct rsa_mpi_key *pkey = rsa_get_key(tfm);
        MPI m, c = mpi_alloc(0);
        int ret = 0;
        int sign;

        if (!c)
                return -ENOMEM;

        if (unlikely(!pkey->n || !pkey->e)) {
                ret = -EINVAL;
                goto err_free_c;
        }

        ret = -ENOMEM;
        m = mpi_read_raw_from_sgl(req->src, req->src_len);
        if (!m)
                goto err_free_c;

        ret = _rsa_enc(pkey, c, m);
        if (ret)
                goto err_free_m;

        ret = mpi_write_to_sgl(c, req->dst, req->dst_len, &sign);
        if (ret)
                goto err_free_m;

        if (sign < 0)
                ret = -EBADMSG;

err_free_m:
        mpi_free(m);
err_free_c:
        mpi_free(c);
        return ret;
}

static int rsa_dec(struct akcipher_request *req)
{
        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
        const struct rsa_mpi_key *pkey = rsa_get_key(tfm);
        MPI c, m = mpi_alloc(0);
        int ret = 0;
        int sign;

        if (!m)
                return -ENOMEM;

        if (unlikely(!pkey->n || !pkey->d)) {
                ret = -EINVAL;
                goto err_free_m;
        }

        ret = -ENOMEM;
        c = mpi_read_raw_from_sgl(req->src, req->src_len);
        if (!c)
                goto err_free_m;

        ret = _rsa_dec_crt(pkey, m, c);
        if (ret)
                goto err_free_c;

        ret = mpi_write_to_sgl(m, req->dst, req->dst_len, &sign);
        if (ret)
                goto err_free_c;

        if (sign < 0)
                ret = -EBADMSG;
err_free_c:
        mpi_free(c);
err_free_m:
        mpi_free(m);
        return ret;
}

static void rsa_free_mpi_key(struct rsa_mpi_key *key)
{
        mpi_free(key->d);
        mpi_free(key->e);
        mpi_free(key->n);
        mpi_free(key->p);
        mpi_free(key->q);
        mpi_free(key->dp);
        mpi_free(key->dq);
        mpi_free(key->qinv);
        key->d = NULL;
        key->e = NULL;
        key->n = NULL;
        key->p = NULL;
        key->q = NULL;
        key->dp = NULL;
        key->dq = NULL;
        key->qinv = NULL;
}

static int rsa_check_key_length(unsigned int len)
{
        switch (len) {
        case 512:
        case 1024:
        case 1536:
                if (fips_enabled)
                        return -EINVAL;
                fallthrough;
        case 2048:
        case 3072:
        case 4096:
                return 0;
        }

        return -EINVAL;
}

static int rsa_check_exponent_fips(MPI e)
{
        MPI e_max = NULL;
        int err;

        /* check if odd */
        if (!mpi_test_bit(e, 0)) {
                return -EINVAL;
        }

        /* check if 2^16 < e < 2^256. */
        if (mpi_cmp_ui(e, 65536) <= 0) {
                return -EINVAL;
        }

        e_max = mpi_alloc(0);
        if (!e_max)
                return -ENOMEM;

        err = mpi_set_bit(e_max, 256);
        if (err) {
                mpi_free(e_max);
                return err;
        }

        if (mpi_cmp(e, e_max) >= 0) {
                mpi_free(e_max);
                return -EINVAL;
        }

        mpi_free(e_max);
        return 0;
}

static int rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
                           unsigned int keylen)
{
        struct rsa_mpi_key *mpi_key = akcipher_tfm_ctx(tfm);
        struct rsa_key raw_key = {0};
        int ret;

        /* Free the old MPI key if any */
        rsa_free_mpi_key(mpi_key);

        ret = rsa_parse_pub_key(&raw_key, key, keylen);
        if (ret)
                return ret;

        mpi_key->e = mpi_read_raw_data(raw_key.e, raw_key.e_sz);
        if (!mpi_key->e)
                goto err;

        mpi_key->n = mpi_read_raw_data(raw_key.n, raw_key.n_sz);
        if (!mpi_key->n)
                goto err;

        if (rsa_check_key_length(mpi_get_size(mpi_key->n) << 3)) {
                rsa_free_mpi_key(mpi_key);
                return -EINVAL;
        }

        if (fips_enabled && rsa_check_exponent_fips(mpi_key->e)) {
                rsa_free_mpi_key(mpi_key);
                return -EINVAL;
        }

        return 0;

err:
        rsa_free_mpi_key(mpi_key);
        return -ENOMEM;
}

static int rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
                            unsigned int keylen)
{
        struct rsa_mpi_key *mpi_key = akcipher_tfm_ctx(tfm);
        struct rsa_key raw_key = {0};
        int ret;

        /* Free the old MPI key if any */
        rsa_free_mpi_key(mpi_key);

        ret = rsa_parse_priv_key(&raw_key, key, keylen);
        if (ret)
                return ret;

        mpi_key->d = mpi_read_raw_data(raw_key.d, raw_key.d_sz);
        if (!mpi_key->d)
                goto err;

        mpi_key->e = mpi_read_raw_data(raw_key.e, raw_key.e_sz);
        if (!mpi_key->e)
                goto err;

        mpi_key->n = mpi_read_raw_data(raw_key.n, raw_key.n_sz);
        if (!mpi_key->n)
                goto err;

        mpi_key->p = mpi_read_raw_data(raw_key.p, raw_key.p_sz);
        if (!mpi_key->p)
                goto err;

        mpi_key->q = mpi_read_raw_data(raw_key.q, raw_key.q_sz);
        if (!mpi_key->q)
                goto err;

        mpi_key->dp = mpi_read_raw_data(raw_key.dp, raw_key.dp_sz);
        if (!mpi_key->dp)
                goto err;

        mpi_key->dq = mpi_read_raw_data(raw_key.dq, raw_key.dq_sz);
        if (!mpi_key->dq)
                goto err;

        mpi_key->qinv = mpi_read_raw_data(raw_key.qinv, raw_key.qinv_sz);
        if (!mpi_key->qinv)
                goto err;

        if (rsa_check_key_length(mpi_get_size(mpi_key->n) << 3)) {
                rsa_free_mpi_key(mpi_key);
                return -EINVAL;
        }

        if (fips_enabled && rsa_check_exponent_fips(mpi_key->e)) {
                rsa_free_mpi_key(mpi_key);
                return -EINVAL;
        }

        return 0;

err:
        rsa_free_mpi_key(mpi_key);
        return -ENOMEM;
}

static unsigned int rsa_max_size(struct crypto_akcipher *tfm)
{
        struct rsa_mpi_key *pkey = akcipher_tfm_ctx(tfm);

        return mpi_get_size(pkey->n);
}

static void rsa_exit_tfm(struct crypto_akcipher *tfm)
{
        struct rsa_mpi_key *pkey = akcipher_tfm_ctx(tfm);

        rsa_free_mpi_key(pkey);
}

static struct akcipher_alg rsa = {
        .encrypt = rsa_enc,
        .decrypt = rsa_dec,
        .set_priv_key = rsa_set_priv_key,
        .set_pub_key = rsa_set_pub_key,
        .max_size = rsa_max_size,
        .exit = rsa_exit_tfm,
        .base = {
                .cra_name = "rsa",
                .cra_driver_name = "rsa-generic",
                .cra_priority = 100,
                .cra_module = THIS_MODULE,
                .cra_ctxsize = sizeof(struct rsa_mpi_key),
        },
};

static int __init rsa_init(void)
{
        int err;

        err = crypto_register_akcipher(&rsa);
        if (err)
                return err;

        err = crypto_register_template(&rsa_pkcs1pad_tmpl);
        if (err)
                goto err_unregister_rsa;

        err = crypto_register_template(&rsassa_pkcs1_tmpl);
        if (err)
                goto err_unregister_rsa_pkcs1pad;

        return 0;

err_unregister_rsa_pkcs1pad:
        crypto_unregister_template(&rsa_pkcs1pad_tmpl);
err_unregister_rsa:
        crypto_unregister_akcipher(&rsa);
        return err;
}

static void __exit rsa_exit(void)
{
        crypto_unregister_template(&rsassa_pkcs1_tmpl);
        crypto_unregister_template(&rsa_pkcs1pad_tmpl);
        crypto_unregister_akcipher(&rsa);
}

module_init(rsa_init);
module_exit(rsa_exit);
MODULE_ALIAS_CRYPTO("rsa");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("RSA generic algorithm");

























































































































  119 






   41 






   35 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
/* SPDX-License-Identifier: GPL-2.0-or-later */

#include <net/inet_common.h>

enum linux_mptcp_mib_field {
        MPTCP_MIB_NUM = 0,
        MPTCP_MIB_MPCAPABLEPASSIVE,        /* Received SYN with MP_CAPABLE */
        MPTCP_MIB_MPCAPABLEACTIVE,        /* Sent SYN with MP_CAPABLE */
        MPTCP_MIB_MPCAPABLEACTIVEACK,        /* Received SYN/ACK with MP_CAPABLE */
        MPTCP_MIB_MPCAPABLEPASSIVEACK,        /* Received third ACK with MP_CAPABLE */
        MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK,/* Server-side fallback during 3-way handshake */
        MPTCP_MIB_MPCAPABLEACTIVEFALLBACK, /* Client-side fallback during 3-way handshake */
        MPTCP_MIB_MPCAPABLEACTIVEDROP,        /* Client-side fallback due to a MPC drop */
        MPTCP_MIB_MPCAPABLEACTIVEDISABLED, /* Client-side disabled due to past issues */
        MPTCP_MIB_MPCAPABLEENDPATTEMPT,        /* Prohibited MPC to port-based endp */
        MPTCP_MIB_TOKENFALLBACKINIT,        /* Could not init/allocate token */
        MPTCP_MIB_RETRANSSEGS,                /* Segments retransmitted at the MPTCP-level */
        MPTCP_MIB_JOINNOTOKEN,                /* Received MP_JOIN but the token was not found */
        MPTCP_MIB_MPJOINNOIDFOUND,        /* Received MP_JOIN but no local ID was found */
        MPTCP_MIB_JOINSYNRX,                /* Received a SYN + MP_JOIN */
        MPTCP_MIB_JOINSYNBACKUPRX,        /* Received a SYN + MP_JOIN + backup flag */
        MPTCP_MIB_JOINSYNACKRX,                /* Received a SYN/ACK + MP_JOIN */
        MPTCP_MIB_JOINSYNACKBACKUPRX,        /* Received a SYN/ACK + MP_JOIN + backup flag */
        MPTCP_MIB_JOINSYNACKMAC,        /* HMAC was wrong on SYN/ACK + MP_JOIN */
        MPTCP_MIB_MPJOINSYNACKNOMPJOIN,        /* MP_RST: missing MP_JOIN in SYN/ACK */
        MPTCP_MIB_JOINACKRX,                /* Received an ACK + MP_JOIN */
        MPTCP_MIB_JOINACKMAC,                /* HMAC was wrong on ACK + MP_JOIN */
        MPTCP_MIB_MPJOINACKNOMPJOIN,        /* MP_RST: missing MP_JOIN in ACK */
        MPTCP_MIB_MPJOINACKNOCTX,        /* MP_RST: no subflow context on ACK */
        MPTCP_MIB_JOINREJECTED,                /* The PM rejected the JOIN request */
        MPTCP_MIB_MPJOINNOTESTABLISHED,        /* MP_RST: JOIN on not-fully-established msk */
        MPTCP_MIB_JOINSYNTX,                /* Sending a SYN + MP_JOIN */
        MPTCP_MIB_JOINSYNTXCREATSKERR,        /* Not able to create a socket when sending a SYN + MP_JOIN */
        MPTCP_MIB_JOINSYNTXBINDERR,        /* Not able to bind() the address when sending a SYN + MP_JOIN */
        MPTCP_MIB_JOINSYNTXCONNECTERR,        /* Not able to connect() when sending a SYN + MP_JOIN */
        MPTCP_MIB_DSSNOMATCH,                /* Received a new mapping that did not match the previous one */
        MPTCP_MIB_DSSCORRUPTIONFALLBACK,/* DSS corruption detected, fallback */
        MPTCP_MIB_DSSCORRUPTIONRESET,        /* DSS corruption detected, MPJ subflow reset */
        MPTCP_MIB_INFINITEMAPTX,        /* Sent an infinite mapping */
        MPTCP_MIB_INFINITEMAPRX,        /* Received an infinite mapping */
        MPTCP_MIB_DSSTCPMISMATCH,        /* DSS-mapping did not map with TCP's sequence numbers */
        MPTCP_MIB_DATACSUMERR,                /* The data checksum fail */
        MPTCP_MIB_OFOQUEUETAIL,        /* Segments inserted into OoO queue tail */
        MPTCP_MIB_OFOQUEUE,                /* Segments inserted into OoO queue */
        MPTCP_MIB_OFOMERGE,                /* Segments merged in OoO queue */
        MPTCP_MIB_NODSSWINDOW,                /* Segments not in MPTCP windows */
        MPTCP_MIB_DUPDATA,                /* Segments discarded due to duplicate DSS */
        MPTCP_MIB_ADDADDR,                /* Received ADD_ADDR with echo-flag=0 */
        MPTCP_MIB_ADDADDRTX,                /* Sent ADD_ADDR with echo-flag=0 */
        MPTCP_MIB_ADDADDRTXDROP,        /* ADD_ADDR with echo-flag=0 not send due to
                                         * resource exhaustion
                                         */
        MPTCP_MIB_ECHOADD,                /* Received ADD_ADDR with echo-flag=1 */
        MPTCP_MIB_ECHOADDTX,                /* Send ADD_ADDR with echo-flag=1 */
        MPTCP_MIB_ECHOADDTXDROP,        /* ADD_ADDR with echo-flag=1 not send due
                                         * to resource exhaustion
                                         */
        MPTCP_MIB_PORTADD,                /* Received ADD_ADDR with a port-number */
        MPTCP_MIB_ADDADDRDROP,                /* Dropped incoming ADD_ADDR */
        MPTCP_MIB_JOINPORTSYNRX,        /* Received a SYN MP_JOIN with a different port-number */
        MPTCP_MIB_JOINPORTSYNACKRX,        /* Received a SYNACK MP_JOIN with a different port-number */
        MPTCP_MIB_JOINPORTACKRX,        /* Received an ACK MP_JOIN with a different port-number */
        MPTCP_MIB_MISMATCHPORTSYNRX,        /* Received a SYN MP_JOIN with a mismatched port-number */
        MPTCP_MIB_MISMATCHPORTACKRX,        /* Received an ACK MP_JOIN with a mismatched port-number */
        MPTCP_MIB_RMADDR,                /* Received RM_ADDR */
        MPTCP_MIB_RMADDRDROP,                /* Dropped incoming RM_ADDR */
        MPTCP_MIB_RMADDRTX,                /* Sent RM_ADDR */
        MPTCP_MIB_RMADDRTXDROP,                /* RM_ADDR not sent due to resource exhaustion */
        MPTCP_MIB_RMSUBFLOW,                /* Remove a subflow */
        MPTCP_MIB_MPPRIOTX,                /* Transmit a MP_PRIO */
        MPTCP_MIB_MPPRIORX,                /* Received a MP_PRIO */
        MPTCP_MIB_MPFAILTX,                /* Transmit a MP_FAIL */
        MPTCP_MIB_MPFAILRX,                /* Received a MP_FAIL */
        MPTCP_MIB_MPFASTCLOSETX,        /* Transmit a MP_FASTCLOSE */
        MPTCP_MIB_MPFASTCLOSERX,        /* Received a MP_FASTCLOSE */
        MPTCP_MIB_MPRSTTX,                /* Transmit a MP_RST */
        MPTCP_MIB_MPRSTRX,                /* Received a MP_RST */
        MPTCP_MIB_SUBFLOWSTALE,                /* Subflows entered 'stale' status */
        MPTCP_MIB_SUBFLOWRECOVER,        /* Subflows returned to active status after being stale */
        MPTCP_MIB_SNDWNDSHARED,                /* Subflow snd wnd is overridden by msk's one */
        MPTCP_MIB_RCVWNDSHARED,                /* Subflow rcv wnd is overridden by msk's one */
        MPTCP_MIB_RCVWNDCONFLICTUPDATE,        /* subflow rcv wnd is overridden by msk's one due to
                                         * conflict with another subflow while updating msk rcv wnd
                                         */
        MPTCP_MIB_RCVWNDCONFLICT,        /* Conflict with while updating msk rcv wnd */
        MPTCP_MIB_CURRESTAB,                /* Current established MPTCP connections */
        MPTCP_MIB_BLACKHOLE,                /* A blackhole has been detected */
        MPTCP_MIB_MPCAPABLEDATAFALLBACK,        /* Missing DSS/MPC+data on first
                                         * established packet
                                         */
        MPTCP_MIB_MD5SIGFALLBACK,        /* Conflicting TCP option enabled */
        MPTCP_MIB_MD5SIGRESET,                /* MP_RST: MD5SIG enabled on listener */
        MPTCP_MIB_DSSFALLBACK,                /* Bad or missing DSS */
        MPTCP_MIB_DSSRESET,                /* MP_RST: bad data mapping */
        MPTCP_MIB_SIMULTCONNFALLBACK,        /* Simultaneous connect */
        MPTCP_MIB_FALLBACKFAILED,        /* Can't fallback due to msk status */
        MPTCP_MIB_WINPROBE,                /* MPTCP-level zero window probe */
        MPTCP_MIB_BACKLOGDROP,                /* Backlog over memory limit */
        MPTCP_MIB_RCVPRUNED,                /* Dropped due to memory constraints */
        MPTCP_MIB_OFOPRUNED,                /* MPTCP-level OoO queue pruned */
        __MPTCP_MIB_MAX
};

#define LINUX_MIB_MPTCP_MAX        __MPTCP_MIB_MAX
struct mptcp_mib {
        unsigned long mibs[LINUX_MIB_MPTCP_MAX];
};

static inline void MPTCP_ADD_STATS(struct net *net,
                                   enum linux_mptcp_mib_field field,
                                   int val)
{
        if (likely(net->mib.mptcp_statistics))
                SNMP_ADD_STATS(net->mib.mptcp_statistics, field, val);
}

static inline void MPTCP_INC_STATS(struct net *net,
                                   enum linux_mptcp_mib_field field)
{
        if (likely(net->mib.mptcp_statistics))
                SNMP_INC_STATS(net->mib.mptcp_statistics, field);
}

static inline void __MPTCP_INC_STATS(struct net *net,
                                     enum linux_mptcp_mib_field field)
{
        if (likely(net->mib.mptcp_statistics))
                __SNMP_INC_STATS(net->mib.mptcp_statistics, field);
}

static inline void MPTCP_DEC_STATS(struct net *net,
                                   enum linux_mptcp_mib_field field)
{
        if (likely(net->mib.mptcp_statistics))
                SNMP_DEC_STATS(net->mib.mptcp_statistics, field);
}

bool mptcp_mib_alloc(struct net *net);






































































  363 







   12 


    5 
  346 
  335 
   16 






    8 

    2 
    8 




    2 





    8 


  343 

  343 


  343 


  343 


  331 
  339 




  331 
















   95 



   95 


   66 





   29 

   54 





   27 

   18 
   16 








   20 





   13 



    9 
    2 










   68 






   22 























   21 












   18 



    3 







    2 


    4 





   15 



   15 



   12 





   12 




   12 



    9 


    3 





    5 
    3 







    4 




    3 
    2 




























    5 






    2 






    2 







    1 











    8 




    1 




    2 






    2 
















    1 

    2 






    1 













    2 










   17 



   17 











































































































































































































































































































































































































































































































































































































































































































































































































































   43 





   24 

   19 








   23 









    9 


   12 
































































































  157 



   63 


   94 


   23 



   28 


  157 



  157 


  156 
    4 

   85 



    2 

   15 




    3 


    3 









  157 
   63 


   63 


   63 


   63 


   63 


   63 


   63 


   63 


   63 


   63 


   63 


   63 


   63 


   63 


   63 


   63 


   63 



   63 


   63 


   63 


   63 


   63 


   63 







  110 







   63 














  171 












  171 
  171 
  171 
  171 
  171 
  171 
  171 
  171 
  171 
  171 
  171 























   88 






   72 


    1 












   11 











   11 



    6 
    5 

    7 
    4 

    6 
    5 



   10 
    1 



    8 
    3 

    9 
    2 

    8 










    3 



    9 
    2 

   11 









    8 




    5 
    3 



    8 


    6 
    2 




    5 
    3 


    5 
    3 

    8 


    8 

















   18 







   10 

    8 

   10 




    2 





    3 
    3 





   13 


















    2 
    2 












    1 





























    1 


















































    1 
    1 








    1 












































  147 



    4 



  158 
  158 





  158 









  136 












  136 





    1 

  136 


  136 


  135 

  135 



    2 



  135 

    3 



  135 















  140 









   31 


  130 



  136 


  139 



















































  140 


  140 






















































   39 


   39 
























   61 






    1 


    5 


   61 





   39 




   61 



   39 



   61 

   56 




    5 


    1 

   61 





   39 

   61 





   61 





   61 










  156 





  156 




    1 







  156 







  148 






    2 


   61 



    1 


   61 



   61 


   60 



   61 









    1 


  154 

    1 



    3 


  154 




  154 


  154 


  154 


    1 











   44 





    1 



    2 


   43 

    1 



   43 
   43 






   43 


   41 


    1 


   41 

    1 



   41 
   41 






   41 

   40 















  148 







  148 
  148 
  148 
  148 
  148 
  148 
  147 
  147 
  147 
  146 
  146 
  146 
  145 
  145 
  145 
  145 
  145 
    3 
  142 
  145 
  145 
  145 
    3 

  142 


  145 


  142 


    3 
  142 
  142 

  145 

  145 
  145 
    3 

  142 


  144 
  143 




  143 

  143 



  143 

  143 














    3 

















































































































































































   36 



   36 














    3 



    3 







    3 











   36 




   36 


































   36 


   36 






































   36 




   36 










































   36 


    1 










   35 





   35 


















    1 


   14 




   14 




   14 












   14 


   14 
    2 





   14 








   35 



   35 






   14 
   14 


   13 

   13 







    3 




   35 





   35 

























    6 














    6 














   35 






   35 



   35 




    1 

    1 

    2 







   35 




    4 














  160 












   32 





   21 

  160 




    5 


   63 























  135 


  135 


  135 


  135 


  135 


  135 



  135 












  135 









  135 





  135 

















  135 







  135 





   20 


  154 












   60 



   11 


   63 

   31 



  154 






   10 


  155 




    9 




   46 





   63 











   94 

   63 




  147 







  158 
   54 
  158 





    1 



   25 

   95 
   48 
   48 
    1 
   47 
   47 
   47 


   47 
   46 
   46 
   46 

   46 





  141 



    3 
  137 





  140 




  140 







  140 










   11 






  139 



  139 















  139 





  139 
















   44 

   95 


   19 



    1 

    2 




   40 


    2 





   39 









   39 




    1 







   38 



   38 








   39 







    1 










   38 


   38 


    1 










   38 


   38 




   39 




   39 



   39 




    1 




    1 




    1 


   38 


   38 


   37 


   37 


   37 


   37 


   36 


   36 




   36 






















   36 








    8 












    1 





    8 








    8 










    8 
    8 








    1 





   35 













   35 







   36 


   36 










   36 












   36 
    8 

   36 







    3 


   35 


    4 



   35 







   62 






   43 











   11 





   28 
    4 
   28 
    4 
   26 





    4 



    1 


    2 










   77 





   60 








    2 



   30 


   75 
   75 

   29 

   75 
























   62 
   62 










   72 











   11 





    1 















    1 


















































   42 










   72 



    2 






   69 

    1 










    2 




   68 


















   13 

    1 





   11 

    2 






    1 




   32 

    9 


   12 



   17 

    4 









    5 

    5 

   54 



   63 



    1 






























   64 







   21 







    9 




   34 






















   42 







   41 

   12 





   12 











   37 































































    5 








    1 

    1 

































   76 























    5 



    7 
   67 







    2 
   10 


















   62 

    7 




    2 










   64 









    2 







































   23 
   41 




   41 









   21 













    1 



    1 










   22 



























   18 
    4 





   19 
    3 





   19 


    1 


    2 











   18 
    3 




   20 



    1 




   20 
    1 





   19 




    1 




   19 




    1 





    1 



    1 




   18 

















   11 













    7 
    4 
    8 
    3 
    9 
    2 
   11 






   10 
    1 
   10 
    1 
   10 
    1 
   10 
    1 


    6 




    5 


















   10 













   10 
   10 









   10 



   10 


   10 


   10 



   10 



   10 












  137 



  137 

  137 







  137 








  137 







    6 




  132 



    6 



  132 





  132 


   13 






   87 





   20 









    6 



  132 





  137 

















































   11 



















   10 









    3 
    1 


   10 
   10 


    2 



   10 






   10 
    1 







   10 




















    1 














    1 






















    1 



   12 
   11 


    2 



   10 








   84 



   70 
    2 


   13 








    1 


   79 







    2 


    3 














   70 


    4 


    2 














    7 
    8 




    3 

    3 



    2 
















   69 




    1 











   52 













    9 
   43 




   50 
    2 
















   11 



   39 



    8 




   42 


    1 


    9 




   42 
    8 

   42 


   21 
   35 
















    1 


    4 
   37 




    9 

   37 



   15 



   28 

   12 




    3 



    7 


   28 







   28 






    7 



   14 
   11 

   20 

    2 



    3 

















   25 








   25 



   42 







   42 



   12 



    2 























   10 









    2 




    1 







    1 





    1 

    1 







































































































































































    2 




    2 





    2 


































































































    8 





    8 


    4 



    1 






    3 

    1 







    1 



    2 









    3 























    7 





    7 


    3 





    3 

    1 









    3 









    1 




    1 




    1 











































































    1 





    1 























   18 



    9 



    8 


















   18 






    4 




    5 

































   69 
















    4 








   13 
   11 


























   33 

















   28 







    8 


























    4 





    1 





    1 











   33 



   33 























    3 








   31 


























   33 

   31 




   33 
   33 
















   33 


   33 











    2 



    2 




    2 













    2 


















    2 















   75 










   75 









   61 


   61 




   61 
   61 



   61 



   33 





   14 



   31 

   33 










    2 






   17 



    4 


    2 





    3 

   38 
   24 



   24 



   34 
   24 







   21 
   28 






   28 


    5 


   27 
    4 





   24 

    3 
   24 

    3 

   25 
    2 




   24 

    1 
   24 

    1 





   12 

    2 
    2 



    1 



    9 



    2 



    2 














    3 

    3 






    3 
    3 

    3 









    3 
    3 

    3 

    1 







    2 
    2 

    2 









    2 
    2 

    2 









    1 


    1 



    1 



    1 








































































































































































































    3 

























    3 


    3 


    3 













    3 








    3 








    3 
















    3 












    4 





    4 









    4 








    3 
    1 



    4 






    4 






    4 














    1 
















    3 


    3 







    3 




























    3 



    3 


    3 


    3 






























































































































    3 





    3 








    3 








    3 








    3 












    3 




























































    4 


































    3 

















    1 

    2 
    3 



















































    4 



    4 














    7 


    7 





    7 






    6 






    1 




    5 

































    2 






    3 





    5 








    1 





    4 






    5 








    5 













    4 


    1 















    4 














    1 


















    3 














    3 




    3 











    3 







    3 







    3 







    3 










    3 




    3 






















    2 
    1 

    3 










    3 


    3 


    3 







    5 























































































    1 

    1 

















    1 













































    1 


































































































































































































    1 



    1 








































































































































































































   32 




















   32 
   32 
   32 


   32 





   32 




   32 
   32 
   32 
   32 

   32 





   32 
   32 




   32 





   32 





   32 




   32 





   32 
   32 
   32 
   32 
   32 
   32 

   32 
   32 
   32 
   32 
   32 
   32 
   32 
   32 
   32 
   32 






   32 

   32 

   32 







   32 

   32 





   32 
   32 
   32 
   32 
   32 
   32 
















   32 



















   32 
   32 
   32 
   32 



   32 







   32 















   31 









   31 









   31 




   31 




   31 








































































































































































































































































    8 













    4 














    1 


    2 





    2 




    3 





    2 



    1 











    3 
















    2 



    1 



    1 
    3 





    1 




























    1 






















    1 






























    3 
    3 




    8 


    8 


    5 
    3 
    3 





    3 












    1 







    2 









    2 









    1 









    1 





    1 








    1 






































    1 


    1 


















    1 











    1 

    1 








    1 




























    1 

























































    1 




    1 





































    1 





























    1 













    1 










    1 


    1 

    1 


    1 


    1 



















    1 









    1 



    1 



    1 









    1 


























    2 









    2 











    2 


    2 


    2 





    2 





    1 





    1 



















    1 






    1 





    1 









    1 



    1 








    1 




    1 




    1 



    1 





    1 



    1 




    1 














    1 

























    5 











    5 





    1 

















    1 


    2 


    1 





    1 









    1 




    1 






    1 

    1 











    1 




    1 


    1 





    1 






    1 



    1 



    1 



















    1 









    1 





    1 



    1 



    1 





    1 





    1 

    1 

















































































































































































    1 




    1 



    1 


    1 


    1 






















































































































   13 









   12 









    1 































































































    1 





    1 













































































    6 









    4 









    2 





    1 

    1 






















    4 








    4 










    4 






    3 




    1 


    3 




    1 


    4 









    3 



    1 


    2 



    2 



    2 
    2 









    1 


    1 













    1 











   13 















    5 


    5 


    3 


    6 


    1 
    2 














   29 






















































































































































































    4 






















    3 
    1 
    4 


    4 


    4 


    4 

    3 
    1 
    4 

    3 

    1 
    4 



    4 


    4 


    4 


    4 


    4 



    4 



    4 


    3 

    1 
    4 



    4 

    4 


    4 


    4 

    4 


    4 


    4 






    4 













    4 



    4 



    3 

    1 
    4 



    4 

    4 

    4 

    3 
    1 
    4 


















    1 







    1 



    1 





    1 





    1 





    1 


    1 








































    7 




    7 







    7 















    7 














    1 




    7 







    7 







    7 























    4 







    2 









    2 



    1 


    3 





    3 




    3 

















    4 



    1 


    4 

    1 


    4 








    4 







    1 





    5 




    5 

    1 


    4 
    4 





    5 

    5 


    1 




    4 








    5 























































   19 








    3 


    1 




    9 
    6 

   11 







   15 



























































































































































































   13 




   13 




   10 












    6 







    6 


























    4 










   10 
    6 




    1 
    3 
    4 


    4 


    3 

    1 
    3 


    2 

    1 
    2 


    2 


    2 


    2 




    2 







































   12 





































   12 


    6 
    6 


    6 


    7 




    5 











    6 
    6 

   10 
    2 






























   12 


   12 

   12 




   12 





   12 












   12 



   12 









    6 
    6 
    1 




    6 





    7 
    4 






   11 

   11 

   10 

















    1 




    5 

    5 





    2 

















    5 

    3 






    8 







    2 


    7 








































































































































































































































































































































































































































































    1 







    1 






































































    1 



    1 








    1 













    1 










































































































































    7 













    2 


    1 










    2 

    2 














    1 





















    1 














    1 





















    1 





























































    1 








    1 


    1 

































    1 






    1 



    1 









    1 



    1 

    1 






    1 




    1 





    1 
    1 



    1 





    1 










    1 






    1 












    1 





























    1 











    1 


















   12 














   10 


















    2 



    2 


    1 


    1 





























    1 














    2 




    2 


    2 


    2 



    2 



    2 



    2 



    2 



    2 



    2 















    4 














    2 



















    2 
    2 





    2 





    2 

















    5 











    1 


    1 


    1 


    1 



















    1 




















    1 






























































































    1 

    1 















   12 
















   12 

   12 











   11 













    1 

    1 






   12 







   12 



   12 
















   12 










   12 












   12 
























































































































    2 







    2 



    1 





    1 



































































































































































































































    2 







    2 



    1 


    1 





























    1 







    1 






    1 
































    5 








    5 


    5 

    3 










   11 









    3 





    8 











    2 




    4 


    3 





    7 






    1 


    1 



    5 

























    5 













    5 




    5 







    5 




    4 





    1 











    4 















    3 
    2 






    1 









    1 



    5 





    5 









    1 



    1 


    3 














































































































































































































































































    8 










    1 






    4 



    3 



    7 










    7 






    4 





    3 

    1 





    4 











    4 


    4 



    1 
    3 

    4 




    3 


    1 







    4 








    4 






    3 
    1 

    4 




    4 









    4 


    4 


    4 


    4 


    4 




    4 









    4 












    4 




















    4 





















    4 





    4 









    4 







    4 















































































































    6 





    1 


    1 
    4 







    3 

















    2 






























































    3 
















    3 




    1 



    1 

    1 
    1 








    1 






    1 




























































    1 









    1 











    2 

    2 















    2 















    1 








    1 








































    1 












    2 




    1 







    1 




   59 

   59 










   42 


   19 



   12 




    1 


    5 
    8 

   11 






























   10 















   10 












    1 





    9 





























    9 















    9 








    9 






    9 


    9 



    9 












    9 













    8 








    6 







    5 





















































    3 







    1 











    1 


    1 
    1 
    1 




    1 
































    1 



























    2 


    2 


















































































    6 
    3 





    5 



    2 


    2 


    1 












    1 











    1 







    1 














    1 






   20 





    1 





    1 


    9 






    1 


    8 



    2 
    3 





    4 






    2 










    2 











   31 










   27 


    4 



    4 






   27 




   26 










   26 






   25 

    1 




   26 





   15 



   11 


   22 





    1 


    3 







    9 




    9 


    2 




    3 






    8 


    1 






    7 
    1 





















































































































































































    2 





    2 



















































































































































































































































    1 









    1 
























































































































































































































































    1 




    1 























    5 



    5 




































































































    1 






    1 

































































































    2 



    2 












    1 










    1 
























































    4 














    2 






    4 





    2 





    1 







    1 



















    1 



    1 





































































































































































































































    1 





    1 



























    1 



    1 
















































































































































































































































































































































    1 






    1 
















    1 




    1 

















































































































































































































































































































































































































































































































































































































































































    1 
















    1 








    1 




    1 




















































































    1 







   10 







    1 




    1 


    8 



    4 





    2 






    2 






    3 



    3 










    1 



    1 


    1 

































































   10 


    4 


















    2 




    2 













































   19 










   16 
    3 

































































































































































































































































































































    2 









    2 






    1 











































































































































































    1 










    1 









































































































































    1 






   11 

    1 

    1 






    9 
    2 




    9 




    2 



    9 








    7 
    2 




    7 
    2 




    7 
    2 




    6 




    1 







    2 










    7 








   16 


   16 







    1 





   15 









   15 


    4 







   13 
   13 







    4 






























































































































    1 





    1 















    1 

    1 




    1 












































































































































































































































































































    1 





    1 













































































































  543 














   69 
    9 



  145 




   30 




  318 













  318 








  437 








   93 







   93 





  525 
    1 






  195 
  333 



  115 
  430 




  108 






  526 
  526 

  214 



   83 

  238 



  195 





  333 


  430 
  115 






  495 
   35 







































    1 









    1 


























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   87 



   87 











   87 






  126 











  126 

















   14 




   14 
















   14 




   14 


   14 



   14 




   14 
   14 
















   14 








   14 





   14 




































   12 












   12 





    2 








    2 










    2 



    2 
































    5 























    5 
















    5 











    5 



























    2 
















    2 




    2 



    2 

























    2 





























































































































































































































































































































































































































































































































































































































































    6 




























    6 








    6 

































































































































































   30 








   30 













   30 
















































































































































































































































































































































































































































































































































































































































    2 

















    2 









    2 






    2 































































































    2 




    2 
    2 

    2 






































































































































































































































































































































































































































































































  424 





  105 


  320 

  320 


  320 









  320 



    1 

  320 








  320 

    1 








  320 
































































































    1 

















    1 
















































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127
7128
7129
7130
7131
7132
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
7219
7220
7221
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231
7232
7233
7234
7235
7236
7237
7238
7239
7240
7241
7242
7243
7244
7245
7246
7247
7248
7249
7250
7251
7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317
7318
7319
7320
7321
7322
7323
7324
7325
7326
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342
7343
7344
7345
7346
7347
7348
7349
7350
7351
7352
7353
7354
7355
7356
7357
7358
7359
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385
7386
7387
7388
7389
7390
7391
7392
7393
7394
7395
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412
7413
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
7441
7442
7443
7444
7445
7446
7447
7448
7449
7450
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
7466
7467
7468
7469
7470
7471
7472
7473
7474
7475
7476
7477
7478
7479
7480
7481
7482
7483
7484
7485
7486
7487
7488
7489
7490
7491
7492
7493
7494
7495
7496
7497
7498
7499
7500
7501
7502
7503
7504
7505
7506
7507
7508
7509
7510
7511
7512
7513
7514
7515
7516
7517
7518
7519
7520
7521
7522
7523
7524
7525
7526
7527
7528
7529
7530
7531
7532
7533
7534
7535
7536
7537
7538
7539
7540
7541
7542
7543
7544
7545
7546
7547
7548
7549
7550
7551
7552
7553
7554
7555
7556
7557
7558
7559
7560
7561
7562
7563
7564
7565
7566
7567
7568
7569
7570
7571
7572
7573
7574
7575
7576
7577
7578
7579
7580
7581
7582
7583
7584
7585
7586
7587
7588
7589
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599
7600
7601
7602
7603
7604
7605
7606
7607
7608
7609
7610
7611
7612
7613
7614
7615
7616
7617
7618
7619
7620
7621
7622
7623
7624
7625
7626
7627
7628
7629
7630
7631
7632
7633
7634
7635
7636
7637
7638
7639
7640
7641
7642
7643
7644
7645
7646
7647
7648
7649
7650
7651
7652
7653
7654
7655
7656
7657
7658
7659
7660
7661
7662
7663
7664
7665
7666
7667
7668
7669
7670
7671
7672
7673
7674
7675
7676
7677
7678
7679
7680
7681
7682
7683
7684
7685
7686
7687
7688
7689
7690
7691
7692
7693
7694
7695
7696
7697
7698
7699
7700
7701
7702
7703
7704
7705
7706
7707
7708
7709
7710
7711
7712
7713
7714
7715
7716
7717
7718
7719
7720
7721
7722
7723
7724
7725
7726
7727
7728
7729
7730
7731
7732
7733
7734
7735
7736
7737
7738
7739
7740
7741
7742
7743
7744
7745
7746
7747
7748
7749
7750
7751
7752
7753
7754
7755
7756
7757
7758
7759
7760
7761
7762
7763
7764
7765
7766
7767
7768
7769
7770
7771
7772
7773
7774
7775
7776
7777
7778
7779
7780
7781
7782
7783
7784
7785
7786
7787
7788
7789
7790
7791
7792
7793
7794
7795
7796
7797
7798
7799
7800
7801
7802
7803
7804
7805
7806
7807
7808
7809
7810
7811
7812
7813
7814
7815
7816
7817
7818
7819
7820
7821
7822
7823
7824
7825
7826
7827
7828
7829
7830
7831
7832
7833
7834
7835
7836
7837
7838
7839
7840
7841
7842
7843
7844
7845
7846
7847
7848
7849
7850
7851
7852
7853
7854
7855
7856
7857
7858
7859
7860
7861
7862
7863
7864
7865
7866
7867
7868
7869
7870
7871
7872
7873
7874
7875
7876
7877
7878
7879
7880
7881
7882
7883
7884
7885
7886
7887
7888
7889
7890
7891
7892
7893
7894
7895
7896
7897
7898
7899
7900
7901
7902
7903
7904
7905
7906
7907
7908
7909
7910
7911
7912
7913
7914
7915
7916
7917
7918
7919
7920
7921
7922
7923
7924
7925
7926
7927
7928
7929
7930
7931
7932
7933
7934
7935
7936
7937
7938
7939
7940
7941
7942
7943
7944
7945
7946
7947
7948
7949
7950
7951
7952
7953
7954
7955
7956
7957
7958
7959
7960
7961
7962
7963
7964
7965
7966
7967
7968
7969
7970
7971
7972
7973
7974
7975
7976
7977
7978
7979
7980
7981
7982
7983
7984
7985
7986
7987
7988
7989
7990
7991
7992
7993
7994
7995
7996
7997
7998
7999
8000
8001
8002
8003
8004
8005
8006
8007
8008
8009
8010
8011
8012
8013
8014
8015
8016
8017
8018
8019
8020
8021
8022
8023
8024
8025
8026
8027
8028
8029
8030
8031
8032
8033
8034
8035
8036
8037
8038
8039
8040
8041
8042
8043
8044
8045
8046
8047
8048
8049
8050
8051
8052
8053
8054
8055
8056
8057
8058
8059
8060
8061
8062
8063
8064
8065
8066
8067
8068
8069
8070
8071
8072
8073
8074
8075
8076
8077
8078
8079
8080
8081
8082
8083
8084
8085
8086
8087
8088
8089
8090
8091
8092
8093
8094
8095
8096
8097
8098
8099
8100
8101
8102
8103
8104
8105
8106
8107
8108
8109
8110
8111
8112
8113
8114
8115
8116
8117
8118
8119
8120
8121
8122
8123
8124
8125
8126
8127
8128
8129
8130
8131
8132
8133
8134
8135
8136
8137
8138
8139
8140
8141
8142
8143
8144
8145
8146
8147
8148
8149
8150
8151
8152
8153
8154
8155
8156
8157
8158
8159
8160
8161
8162
8163
8164
8165
8166
8167
8168
8169
8170
8171
8172
8173
8174
8175
8176
8177
8178
8179
8180
8181
8182
8183
8184
8185
8186
8187
8188
8189
8190
8191
8192
8193
8194
8195
8196
8197
8198
8199
8200
8201
8202
8203
8204
8205
8206
8207
8208
8209
8210
8211
8212
8213
8214
8215
8216
8217
8218
8219
8220
8221
8222
8223
8224
8225
8226
8227
8228
8229
8230
8231
8232
8233
8234
8235
8236
8237
8238
8239
8240
8241
8242
8243
8244
8245
8246
8247
8248
8249
8250
8251
8252
8253
8254
8255
8256
8257
8258
8259
8260
8261
8262
8263
8264
8265
8266
8267
8268
8269
8270
8271
8272
8273
8274
8275
8276
8277
8278
8279
8280
8281
8282
8283
8284
8285
8286
8287
8288
8289
8290
8291
8292
8293
8294
8295
8296
8297
8298
8299
8300
8301
8302
8303
8304
8305
8306
8307
8308
8309
8310
8311
8312
8313
8314
8315
8316
8317
8318
8319
8320
8321
8322
8323
8324
8325
8326
8327
8328
8329
8330
8331
8332
8333
8334
8335
8336
8337
8338
8339
8340
8341
8342
8343
8344
8345
8346
8347
8348
8349
8350
8351
8352
8353
8354
8355
8356
8357
8358
8359
8360
8361
8362
8363
8364
8365
8366
8367
8368
8369
8370
8371
8372
8373
8374
8375
8376
8377
8378
8379
8380
8381
8382
8383
8384
8385
8386
8387
8388
8389
8390
8391
8392
8393
8394
8395
8396
8397
8398
8399
8400
8401
8402
8403
8404
8405
8406
8407
8408
8409
8410
8411
8412
8413
8414
8415
8416
8417
8418
8419
8420
8421
8422
8423
8424
8425
8426
8427
8428
8429
8430
8431
8432
8433
8434
8435
8436
8437
8438
8439
8440
8441
8442
8443
8444
8445
8446
8447
8448
8449
8450
8451
8452
8453
8454
8455
8456
8457
8458
8459
8460
8461
8462
8463
8464
8465
8466
8467
8468
8469
8470
8471
8472
8473
8474
8475
8476
8477
8478
8479
8480
8481
8482
8483
8484
8485
8486
8487
8488
8489
8490
8491
8492
8493
8494
8495
8496
8497
8498
8499
8500
8501
8502
8503
8504
8505
8506
8507
8508
8509
8510
8511
8512
8513
8514
8515
8516
8517
8518
8519
8520
8521
8522
8523
8524
8525
8526
8527
8528
8529
8530
8531
8532
8533
8534
8535
8536
8537
8538
8539
8540
8541
8542
8543
8544
8545
8546
8547
8548
8549
8550
8551
8552
8553
8554
8555
8556
8557
8558
8559
8560
8561
8562
8563
8564
8565
8566
8567
8568
8569
8570
8571
8572
8573
8574
8575
8576
8577
8578
8579
8580
8581
8582
8583
8584
8585
8586
8587
8588
8589
8590
8591
8592
8593
8594
8595
8596
8597
8598
8599
8600
8601
8602
8603
8604
8605
8606
8607
8608
8609
8610
8611
8612
8613
8614
8615
8616
8617
8618
8619
8620
8621
8622
8623
8624
8625
8626
8627
8628
8629
8630
8631
8632
8633
8634
8635
8636
8637
8638
8639
8640
8641
8642
8643
8644
8645
8646
8647
8648
8649
8650
8651
8652
8653
8654
8655
8656
8657
8658
8659
8660
8661
8662
8663
8664
8665
8666
8667
8668
8669
8670
8671
8672
8673
8674
8675
8676
8677
8678
8679
8680
8681
8682
8683
8684
8685
8686
8687
8688
8689
8690
8691
8692
8693
8694
8695
8696
8697
8698
8699
8700
8701
8702
8703
8704
8705
8706
8707
8708
8709
8710
8711
8712
8713
8714
8715
8716
8717
8718
8719
8720
8721
8722
8723
8724
8725
8726
8727
8728
8729
8730
8731
8732
8733
8734
8735
8736
8737
8738
8739
8740
8741
8742
8743
8744
8745
8746
8747
8748
8749
8750
8751
8752
8753
8754
8755
8756
8757
8758
8759
8760
8761
8762
8763
8764
8765
8766
8767
8768
8769
8770
8771
8772
8773
8774
8775
8776
8777
8778
8779
8780
8781
8782
8783
8784
8785
8786
8787
8788
8789
8790
8791
8792
8793
8794
8795
8796
8797
8798
8799
8800
8801
8802
8803
8804
8805
8806
8807
8808
8809
8810
8811
8812
8813
8814
8815
8816
8817
8818
8819
8820
8821
8822
8823
8824
8825
8826
8827
8828
8829
8830
8831
8832
8833
8834
8835
8836
8837
8838
8839
8840
8841
8842
8843
8844
8845
8846
8847
8848
8849
8850
8851
8852
8853
8854
8855
8856
8857
8858
8859
8860
8861
8862
8863
8864
8865
8866
8867
8868
8869
8870
8871
8872
8873
8874
8875
8876
8877
8878
8879
8880
8881
8882
8883
8884
8885
8886
8887
8888
8889
8890
8891
8892
8893
8894
8895
8896
8897
8898
8899
8900
8901
8902
8903
8904
8905
8906
8907
8908
8909
8910
8911
8912
8913
8914
8915
8916
8917
8918
8919
8920
8921
8922
8923
8924
8925
8926
8927
8928
8929
8930
8931
8932
8933
8934
8935
8936
8937
8938
8939
8940
8941
8942
8943
8944
8945
8946
8947
8948
8949
8950
8951
8952
8953
8954
8955
8956
8957
8958
8959
8960
8961
8962
8963
8964
8965
8966
8967
8968
8969
8970
8971
8972
8973
8974
8975
8976
8977
8978
8979
8980
8981
8982
8983
8984
8985
8986
8987
8988
8989
8990
8991
8992
8993
8994
8995
8996
8997
8998
8999
9000
9001
9002
9003
9004
9005
9006
9007
9008
9009
9010
9011
9012
9013
9014
9015
9016
9017
9018
9019
9020
9021
9022
9023
9024
9025
9026
9027
9028
9029
9030
9031
9032
9033
9034
9035
9036
9037
9038
9039
9040
9041
9042
9043
9044
9045
9046
9047
9048
9049
9050
9051
9052
9053
9054
9055
9056
9057
9058
9059
9060
9061
9062
9063
9064
9065
9066
9067
9068
9069
9070
9071
9072
9073
9074
9075
9076
9077
9078
9079
9080
9081
9082
9083
9084
9085
9086
9087
9088
9089
9090
9091
9092
9093
9094
9095
9096
9097
9098
9099
9100
9101
9102
9103
9104
9105
9106
9107
9108
9109
9110
9111
9112
9113
9114
9115
9116
9117
9118
9119
9120
9121
9122
9123
9124
9125
9126
9127
9128
9129
9130
9131
9132
9133
9134
9135
9136
9137
9138
9139
9140
9141
9142
9143
9144
9145
9146
9147
9148
9149
9150
9151
9152
9153
9154
9155
9156
9157
9158
9159
9160
9161
9162
9163
9164
9165
9166
9167
9168
9169
9170
9171
9172
9173
9174
9175
9176
9177
9178
9179
9180
9181
9182
9183
9184
9185
9186
9187
9188
9189
9190
9191
9192
9193
9194
9195
9196
9197
9198
9199
9200
9201
9202
9203
9204
9205
9206
9207
9208
9209
9210
9211
9212
9213
9214
9215
9216
9217
9218
9219
9220
9221
9222
9223
9224
9225
9226
9227
9228
9229
9230
9231
9232
9233
9234
9235
9236
9237
9238
9239
9240
9241
9242
9243
9244
9245
9246
9247
9248
9249
9250
9251
9252
9253
9254
9255
9256
9257
9258
9259
9260
9261
9262
9263
9264
9265
9266
9267
9268
9269
9270
9271
9272
9273
9274
9275
9276
9277
9278
9279
9280
9281
9282
9283
9284
9285
9286
9287
9288
9289
9290
9291
9292
9293
9294
9295
9296
9297
9298
9299
9300
9301
9302
9303
9304
9305
9306
9307
9308
9309
9310
9311
9312
9313
9314
9315
9316
9317
9318
9319
9320
9321
9322
9323
9324
9325
9326
9327
9328
9329
9330
9331
9332
9333
9334
9335
9336
9337
9338
9339
9340
9341
9342
9343
9344
9345
9346
9347
9348
9349
9350
9351
9352
9353
9354
9355
9356
9357
9358
9359
9360
9361
9362
9363
9364
9365
9366
9367
9368
9369
9370
9371
9372
9373
9374
9375
9376
9377
9378
9379
9380
9381
9382
9383
9384
9385
9386
9387
9388
9389
9390
9391
9392
9393
9394
9395
9396
9397
9398
9399
9400
9401
9402
9403
9404
9405
9406
9407
9408
9409
9410
9411
9412
9413
9414
9415
9416
9417
9418
9419
9420
9421
9422
9423
9424
9425
9426
9427
9428
9429
9430
9431
9432
9433
9434
9435
9436
9437
9438
9439
9440
9441
9442
9443
9444
9445
9446
9447
9448
9449
9450
9451
9452
9453
9454
9455
9456
9457
9458
9459
9460
9461
9462
9463
9464
9465
9466
9467
9468
9469
9470
9471
9472
9473
9474
9475
9476
9477
9478
9479
9480
9481
9482
9483
9484
9485
9486
9487
9488
9489
9490
9491
9492
9493
9494
9495
9496
9497
9498
9499
9500
9501
9502
9503
9504
9505
9506
9507
9508
9509
9510
9511
9512
9513
9514
9515
9516
9517
9518
9519
9520
9521
9522
9523
9524
9525
9526
9527
9528
9529
9530
9531
9532
9533
9534
9535
9536
9537
9538
9539
9540
9541
9542
9543
9544
9545
9546
9547
9548
9549
9550
9551
9552
9553
9554
9555
9556
9557
9558
9559
9560
9561
9562
9563
9564
9565
9566
9567
9568
9569
9570
9571
9572
9573
9574
9575
9576
9577
9578
9579
9580
9581
9582
9583
9584
9585
9586
9587
9588
9589
9590
9591
9592
9593
9594
9595
9596
9597
9598
9599
9600
9601
9602
9603
9604
9605
9606
9607
9608
9609
9610
9611
9612
9613
9614
9615
9616
9617
9618
9619
9620
9621
9622
9623
9624
9625
9626
9627
9628
9629
9630
9631
9632
9633
9634
9635
9636
9637
9638
9639
9640
9641
9642
9643
9644
9645
9646
9647
9648
9649
9650
9651
9652
9653
9654
9655
9656
9657
9658
9659
9660
9661
9662
9663
9664
9665
9666
9667
9668
9669
9670
9671
9672
9673
9674
9675
9676
9677
9678
9679
9680
9681
9682
9683
9684
9685
9686
9687
9688
9689
9690
9691
9692
9693
9694
9695
9696
9697
9698
9699
9700
9701
9702
9703
9704
9705
9706
9707
9708
9709
9710
9711
9712
9713
9714
9715
9716
9717
9718
9719
9720
9721
9722
9723
9724
9725
9726
9727
9728
9729
9730
9731
9732
9733
9734
9735
9736
9737
9738
9739
9740
9741
9742
9743
9744
9745
9746
9747
9748
9749
9750
9751
9752
9753
9754
9755
9756
9757
9758
9759
9760
9761
9762
9763
9764
9765
9766
9767
9768
9769
9770
9771
9772
9773
9774
9775
9776
9777
9778
9779
9780
9781
9782
9783
9784
9785
9786
9787
9788
9789
9790
9791
9792
9793
9794
9795
9796
9797
9798
9799
9800
9801
9802
9803
9804
9805
9806
9807
9808
9809
9810
9811
9812
9813
9814
9815
9816
9817
9818
9819
9820
9821
9822
9823
9824
9825
9826
9827
9828
9829
9830
9831
9832
9833
9834
9835
9836
9837
9838
9839
9840
9841
9842
9843
9844
9845
9846
9847
9848
9849
9850
9851
9852
9853
9854
9855
9856
9857
9858
9859
9860
9861
9862
9863
9864
9865
9866
9867
9868
9869
9870
9871
9872
9873
9874
9875
9876
9877
9878
9879
9880
9881
9882
9883
9884
9885
9886
9887
9888
9889
9890
9891
9892
9893
9894
9895
9896
9897
9898
9899
9900
9901
9902
9903
9904
9905
9906
9907
9908
9909
9910
9911
9912
9913
9914
9915
9916
9917
9918
9919
9920
9921
9922
9923
9924
9925
9926
9927
9928
9929
9930
9931
9932
9933
9934
9935
9936
9937
9938
9939
9940
9941
9942
9943
9944
9945
9946
9947
9948
9949
9950
9951
9952
9953
9954
9955
9956
9957
9958
9959
9960
9961
9962
9963
9964
9965
9966
9967
9968
9969
9970
9971
9972
9973
9974
9975
9976
9977
9978
9979
9980
9981
9982
9983
9984
9985
9986
9987
9988
9989
9990
9991
9992
9993
9994
9995
9996
9997
9998
9999
10000
10001
10002
10003
10004
10005
10006
10007
10008
10009
10010
10011
10012
10013
10014
10015
10016
10017
10018
10019
10020
10021
10022
10023
10024
10025
10026
10027
10028
10029
10030
10031
10032
10033
10034
10035
10036
10037
10038
10039
10040
10041
10042
10043
10044
10045
10046
10047
10048
10049
10050
10051
10052
10053
10054
10055
10056
10057
10058
10059
10060
10061
10062
10063
10064
10065
10066
10067
10068
10069
10070
10071
10072
10073
10074
10075
10076
10077
10078
10079
10080
10081
10082
10083
10084
10085
10086
10087
10088
10089
10090
10091
10092
10093
10094
10095
10096
10097
10098
10099
10100
10101
10102
10103
10104
10105
10106
10107
10108
10109
10110
10111
10112
10113
10114
10115
10116
10117
10118
10119
10120
10121
10122
10123
10124
10125
10126
10127
10128
10129
10130
10131
10132
10133
10134
10135
10136
10137
10138
10139
10140
10141
10142
10143
10144
10145
10146
10147
10148
10149
10150
10151
10152
10153
10154
10155
10156
10157
10158
10159
10160
10161
10162
10163
10164
10165
10166
10167
10168
10169
10170
10171
10172
10173
10174
10175
10176
10177
10178
10179
10180
10181
10182
10183
10184
10185
10186
10187
10188
10189
10190
10191
10192
10193
10194
10195
10196
10197
10198
10199
10200
10201
10202
10203
10204
10205
10206
10207
10208
10209
10210
10211
10212
10213
10214
10215
10216
10217
10218
10219
10220
10221
10222
10223
10224
10225
10226
10227
10228
10229
10230
10231
10232
10233
10234
10235
10236
10237
10238
10239
10240
10241
10242
10243
10244
10245
10246
10247
10248
10249
10250
10251
10252
10253
10254
10255
10256
10257
10258
10259
10260
10261
10262
10263
10264
10265
10266
10267
10268
10269
10270
10271
10272
10273
10274
10275
10276
10277
10278
10279
10280
10281
10282
10283
10284
10285
10286
10287
10288
10289
10290
10291
10292
10293
10294
10295
10296
10297
10298
10299
10300
10301
10302
10303
10304
10305
10306
10307
10308
10309
10310
10311
10312
10313
10314
10315
10316
10317
10318
10319
10320
10321
10322
10323
10324
10325
10326
10327
10328
10329
10330
10331
10332
10333
10334
10335
10336
10337
10338
10339
10340
10341
10342
10343
10344
10345
10346
10347
10348
10349
10350
10351
10352
10353
10354
10355
10356
10357
10358
10359
10360
10361
10362
10363
10364
10365
10366
10367
10368
10369
10370
10371
10372
10373
10374
10375
10376
10377
10378
10379
10380
10381
10382
10383
10384
10385
10386
10387
10388
10389
10390
10391
10392
10393
10394
10395
10396
10397
10398
10399
10400
10401
10402
10403
10404
10405
10406
10407
10408
10409
10410
10411
10412
10413
10414
10415
10416
10417
10418
10419
10420
10421
10422
10423
10424
10425
10426
10427
10428
10429
10430
10431
10432
10433
10434
10435
10436
10437
10438
10439
10440
10441
10442
10443
10444
10445
10446
10447
10448
10449
10450
10451
10452
10453
10454
10455
10456
10457
10458
10459
10460
10461
10462
10463
10464
10465
10466
10467
10468
10469
10470
10471
10472
10473
10474
10475
10476
10477
10478
10479
10480
10481
10482
10483
10484
10485
10486
10487
10488
10489
10490
10491
10492
10493
10494
10495
10496
10497
10498
10499
10500
10501
10502
10503
10504
10505
10506
10507
10508
10509
10510
10511
10512
10513
10514
10515
10516
10517
10518
10519
10520
10521
10522
10523
10524
10525
10526
10527
10528
10529
10530
10531
10532
10533
10534
10535
10536
10537
10538
10539
10540
10541
10542
10543
10544
10545
10546
10547
10548
10549
10550
10551
10552
10553
10554
10555
10556
10557
10558
10559
10560
10561
10562
10563
10564
10565
10566
10567
10568
10569
10570
10571
10572
10573
10574
10575
10576
10577
10578
10579
10580
10581
10582
10583
10584
10585
10586
10587
10588
10589
10590
10591
10592
10593
10594
10595
10596
10597
10598
10599
10600
10601
10602
10603
10604
10605
10606
10607
10608
10609
10610
10611
10612
10613
10614
10615
10616
10617
10618
10619
10620
10621
10622
10623
10624
10625
10626
10627
10628
10629
10630
10631
10632
10633
10634
10635
10636
10637
10638
10639
10640
10641
10642
10643
10644
10645
10646
10647
10648
10649
10650
10651
10652
10653
10654
10655
10656
10657
10658
10659
10660
10661
10662
10663
10664
10665
10666
10667
10668
10669
10670
10671
10672
10673
10674
10675
10676
10677
10678
10679
10680
10681
10682
10683
10684
10685
10686
10687
10688
10689
10690
10691
10692
10693
10694
10695
10696
10697
10698
10699
10700
10701
10702
10703
10704
10705
10706
10707
10708
10709
10710
10711
10712
10713
10714
10715
10716
10717
10718
10719
10720
10721
10722
10723
10724
10725
10726
10727
10728
10729
10730
10731
10732
10733
10734
10735
10736
10737
10738
10739
10740
10741
10742
10743
10744
10745
10746
10747
10748
10749
10750
10751
10752
10753
10754
10755
10756
10757
10758
10759
10760
10761
10762
10763
10764
10765
10766
10767
10768
10769
10770
10771
10772
10773
10774
10775
10776
10777
10778
10779
10780
10781
10782
10783
10784
10785
10786
10787
10788
10789
10790
10791
10792
10793
10794
10795
10796
10797
10798
10799
10800
10801
10802
10803
10804
10805
10806
10807
10808
10809
10810
10811
10812
10813
10814
10815
10816
10817
10818
10819
10820
10821
10822
10823
10824
10825
10826
10827
10828
10829
10830
10831
10832
10833
10834
10835
10836
10837
10838
10839
10840
10841
10842
10843
10844
10845
10846
10847
10848
10849
10850
10851
10852
10853
10854
10855
10856
10857
10858
10859
10860
10861
10862
10863
10864
10865
10866
10867
10868
10869
10870
10871
10872
10873
10874
10875
10876
10877
10878
10879
10880
10881
10882
10883
10884
10885
10886
10887
10888
10889
10890
10891
10892
10893
10894
10895
10896
10897
10898
10899
10900
10901
10902
10903
10904
10905
10906
10907
10908
10909
10910
10911
10912
10913
10914
10915
10916
10917
10918
10919
10920
10921
10922
10923
10924
10925
10926
10927
10928
10929
10930
10931
10932
10933
10934
10935
10936
10937
10938
10939
10940
10941
10942
10943
10944
10945
10946
10947
10948
10949
10950
10951
10952
10953
10954
10955
10956
10957
10958
10959
10960
10961
10962
10963
10964
10965
10966
10967
10968
10969
10970
10971
10972
10973
10974
10975
10976
10977
10978
10979
10980
10981
10982
10983
10984
10985
10986
10987
10988
10989
10990
10991
10992
10993
10994
10995
10996
10997
10998
10999
11000
11001
11002
11003
11004
11005
11006
11007
11008
11009
11010
11011
11012
11013
11014
11015
11016
11017
11018
11019
11020
11021
11022
11023
11024
11025
11026
11027
11028
11029
11030
11031
11032
11033
11034
11035
11036
11037
11038
11039
11040
11041
11042
11043
11044
11045
11046
11047
11048
11049
11050
11051
11052
11053
11054
11055
11056
11057
11058
11059
11060
11061
11062
11063
11064
11065
11066
11067
11068
11069
11070
11071
11072
11073
11074
11075
11076
11077
11078
11079
11080
11081
11082
11083
11084
11085
11086
11087
11088
11089
11090
11091
11092
11093
11094
11095
11096
11097
11098
11099
11100
11101
11102
11103
11104
11105
11106
11107
11108
11109
11110
11111
11112
11113
11114
11115
11116
11117
11118
11119
11120
11121
11122
11123
11124
11125
11126
11127
11128
11129
11130
11131
11132
11133
11134
11135
11136
11137
11138
11139
11140
11141
11142
11143
11144
11145
11146
11147
11148
11149
11150
11151
11152
11153
11154
11155
11156
11157
11158
11159
11160
11161
11162
11163
11164
11165
11166
11167
11168
11169
11170
11171
11172
11173
11174
11175
11176
11177
11178
11179
11180
11181
11182
11183
11184
11185
11186
11187
11188
11189
11190
11191
11192
11193
11194
11195
11196
11197
11198
11199
11200
11201
11202
11203
11204
11205
11206
11207
11208
11209
11210
11211
11212
11213
11214
11215
11216
11217
11218
11219
11220
11221
11222
11223
11224
11225
11226
11227
11228
11229
11230
11231
11232
11233
11234
11235
11236
11237
11238
11239
11240
11241
11242
11243
11244
11245
11246
11247
11248
11249
11250
11251
11252
11253
11254
11255
11256
11257
11258
11259
11260
11261
11262
11263
11264
11265
11266
11267
11268
11269
11270
11271
11272
11273
11274
11275
11276
11277
11278
11279
11280
11281
11282
11283
11284
11285
11286
11287
11288
11289
11290
11291
11292
11293
11294
11295
11296
11297
11298
11299
11300
11301
11302
11303
11304
11305
11306
11307
11308
11309
11310
11311
11312
11313
11314
11315
11316
11317
11318
11319
11320
11321
11322
11323
11324
11325
11326
11327
11328
11329
11330
11331
11332
11333
11334
11335
11336
11337
11338
11339
11340
11341
11342
11343
11344
11345
11346
11347
11348
11349
11350
11351
11352
11353
11354
11355
11356
11357
11358
11359
11360
11361
11362
11363
11364
11365
11366
11367
11368
11369
11370
11371
11372
11373
11374
11375
11376
11377
11378
11379
11380
11381
11382
11383
11384
11385
11386
11387
11388
11389
11390
11391
11392
11393
11394
11395
11396
11397
11398
11399
11400
11401
11402
11403
11404
11405
11406
11407
11408
11409
11410
11411
11412
11413
11414
11415
11416
11417
11418
11419
11420
11421
11422
11423
11424
11425
11426
11427
11428
11429
11430
11431
11432
11433
11434
11435
11436
11437
11438
11439
11440
11441
11442
11443
11444
11445
11446
11447
11448
11449
11450
11451
11452
11453
11454
11455
11456
11457
11458
11459
11460
11461
11462
11463
11464
11465
11466
11467
11468
11469
11470
11471
11472
11473
11474
11475
11476
11477
11478
11479
11480
11481
11482
11483
11484
11485
11486
11487
11488
11489
11490
11491
11492
11493
11494
11495
11496
11497
11498
11499
11500
11501
11502
11503
11504
11505
11506
11507
11508
11509
11510
11511
11512
11513
11514
11515
11516
11517
11518
11519
11520
11521
11522
11523
11524
11525
11526
11527
11528
11529
11530
11531
11532
11533
11534
11535
11536
11537
11538
11539
11540
11541
11542
11543
11544
11545
11546
11547
11548
11549
11550
11551
11552
11553
11554
11555
11556
11557
11558
11559
11560
11561
11562
11563
11564
11565
11566
11567
11568
11569
11570
11571
11572
11573
11574
11575
11576
11577
11578
11579
11580
11581
11582
11583
11584
11585
11586
11587
11588
11589
11590
11591
11592
11593
11594
11595
11596
11597
11598
11599
11600
11601
11602
11603
11604
11605
11606
11607
11608
11609
11610
11611
11612
11613
11614
11615
11616
11617
11618
11619
11620
11621
11622
11623
11624
11625
11626
11627
11628
11629
11630
11631
11632
11633
11634
11635
11636
11637
11638
11639
11640
11641
11642
11643
11644
11645
11646
11647
11648
11649
11650
11651
11652
11653
11654
11655
11656
11657
11658
11659
11660
11661
11662
11663
11664
11665
11666
11667
11668
11669
11670
11671
11672
11673
11674
11675
11676
11677
11678
11679
11680
11681
11682
11683
11684
11685
11686
11687
11688
11689
11690
11691
11692
11693
11694
11695
11696
11697
11698
11699
11700
11701
11702
11703
11704
11705
11706
11707
11708
11709
11710
11711
11712
11713
11714
11715
11716
11717
11718
11719
11720
11721
11722
11723
11724
11725
11726
11727
11728
11729
11730
11731
11732
11733
11734
11735
11736
11737
11738
11739
11740
11741
11742
11743
11744
11745
11746
11747
11748
11749
11750
11751
11752
11753
11754
11755
11756
11757
11758
11759
11760
11761
11762
11763
11764
11765
11766
11767
11768
11769
11770
11771
11772
11773
11774
11775
11776
11777
11778
11779
11780
11781
11782
11783
11784
11785
11786
11787
11788
11789
11790
11791
11792
11793
11794
11795
11796
11797
11798
11799
11800
11801
11802
11803
11804
11805
11806
11807
11808
11809
11810
11811
11812
11813
11814
11815
11816
11817
11818
11819
11820
11821
11822
11823
11824
11825
11826
11827
11828
11829
11830
11831
11832
11833
11834
11835
11836
11837
11838
11839
11840
11841
11842
11843
11844
11845
11846
11847
11848
11849
11850
11851
11852
11853
11854
11855
11856
11857
11858
11859
11860
11861
11862
11863
11864
11865
11866
11867
11868
11869
11870
11871
11872
11873
11874
11875
11876
11877
11878
11879
11880
11881
11882
11883
11884
11885
11886
11887
11888
11889
11890
11891
11892
11893
11894
11895
11896
11897
11898
11899
11900
11901
11902
11903
11904
11905
11906
11907
11908
11909
11910
11911
11912
11913
11914
11915
11916
11917
11918
11919
11920
11921
11922
11923
11924
11925
11926
11927
11928
11929
11930
11931
11932
11933
11934
11935
11936
11937
11938
11939
11940
11941
11942
11943
11944
11945
11946
11947
11948
11949
11950
11951
11952
11953
11954
11955
11956
11957
11958
11959
11960
11961
11962
11963
11964
11965
11966
11967
11968
11969
11970
11971
11972
11973
11974
11975
11976
11977
11978
11979
11980
11981
11982
11983
11984
11985
11986
11987
11988
11989
11990
11991
11992
11993
11994
11995
11996
11997
11998
11999
12000
12001
12002
12003
12004
12005
12006
12007
12008
12009
12010
12011
12012
12013
12014
12015
12016
12017
12018
12019
12020
12021
12022
12023
12024
12025
12026
12027
12028
12029
12030
12031
12032
12033
12034
12035
12036
12037
12038
12039
12040
12041
12042
12043
12044
12045
12046
12047
12048
12049
12050
12051
12052
12053
12054
12055
12056
12057
12058
12059
12060
12061
12062
12063
12064
12065
12066
12067
12068
12069
12070
12071
12072
12073
12074
12075
12076
12077
12078
12079
12080
12081
12082
12083
12084
12085
12086
12087
12088
12089
12090
12091
12092
12093
12094
12095
12096
12097
12098
12099
12100
12101
12102
12103
12104
12105
12106
12107
12108
12109
12110
12111
12112
12113
12114
12115
12116
12117
12118
12119
12120
12121
12122
12123
12124
12125
12126
12127
12128
12129
12130
12131
12132
12133
12134
12135
12136
12137
12138
12139
12140
12141
12142
12143
12144
12145
12146
12147
12148
12149
12150
12151
12152
12153
12154
12155
12156
12157
12158
12159
12160
12161
12162
12163
12164
12165
12166
12167
12168
12169
12170
12171
12172
12173
12174
12175
12176
12177
12178
12179
12180
12181
12182
12183
12184
12185
12186
12187
12188
12189
12190
12191
12192
12193
12194
12195
12196
12197
12198
12199
12200
12201
12202
12203
12204
12205
12206
12207
12208
12209
12210
12211
12212
12213
12214
12215
12216
12217
12218
12219
12220
12221
12222
12223
12224
12225
12226
12227
12228
12229
12230
12231
12232
12233
12234
12235
12236
12237
12238
12239
12240
12241
12242
12243
12244
12245
12246
12247
12248
12249
12250
12251
12252
12253
12254
12255
12256
12257
12258
12259
12260
12261
12262
12263
12264
12265
12266
12267
12268
12269
12270
12271
12272
12273
12274
12275
12276
12277
12278
12279
12280
12281
12282
12283
12284
12285
12286
12287
12288
12289
12290
12291
12292
12293
12294
12295
12296
12297
12298
12299
12300
12301
12302
12303
12304
12305
12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
12325
12326
12327
12328
12329
12330
12331
12332
12333
12334
12335
12336
12337
12338
12339
12340
12341
12342
12343
12344
12345
12346
12347
12348
12349
12350
12351
12352
12353
12354
12355
12356
12357
12358
12359
12360
12361
12362
12363
12364
12365
12366
12367
12368
12369
12370
12371
12372
12373
12374
12375
12376
12377
12378
12379
12380
12381
12382
12383
12384
12385
12386
12387
12388
12389
12390
12391
12392
12393
12394
12395
12396
12397
12398
12399
12400
12401
12402
12403
12404
12405
12406
12407
12408
12409
12410
12411
12412
12413
12414
12415
12416
12417
12418
12419
12420
12421
12422
12423
12424
12425
12426
12427
12428
12429
12430
12431
12432
12433
12434
12435
12436
12437
12438
12439
12440
12441
12442
12443
12444
12445
12446
12447
12448
12449
12450
12451
12452
12453
12454
12455
12456
12457
12458
12459
12460
12461
12462
12463
12464
12465
12466
12467
12468
12469
12470
12471
12472
12473
12474
12475
12476
12477
12478
12479
12480
12481
12482
12483
12484
12485
12486
12487
12488
12489
12490
12491
12492
12493
12494
12495
12496
12497
12498
12499
12500
12501
12502
12503
12504
12505
12506
12507
12508
12509
12510
12511
12512
12513
12514
12515
12516
12517
12518
12519
12520
12521
12522
12523
12524
12525
12526
12527
12528
12529
12530
12531
12532
12533
12534
12535
12536
12537
12538
12539
12540
12541
12542
12543
12544
12545
12546
12547
12548
12549
12550
12551
12552
12553
12554
12555
12556
12557
12558
12559
12560
12561
12562
12563
12564
12565
12566
12567
12568
12569
12570
12571
12572
12573
12574
12575
12576
12577
12578
12579
12580
12581
12582
12583
12584
12585
12586
12587
12588
12589
12590
12591
12592
12593
12594
12595
12596
12597
12598
12599
12600
12601
12602
12603
12604
12605
12606
12607
12608
12609
12610
12611
12612
12613
12614
12615
12616
12617
12618
12619
12620
12621
12622
12623
12624
12625
12626
12627
12628
12629
12630
12631
12632
12633
12634
12635
12636
12637
12638
12639
12640
12641
12642
12643
12644
12645
12646
12647
12648
12649
12650
12651
12652
12653
12654
12655
12656
12657
12658
12659
12660
12661
12662
12663
12664
12665
12666
12667
12668
12669
12670
12671
12672
12673
12674
12675
12676
12677
12678
12679
12680
12681
12682
12683
12684
12685
12686
12687
12688
12689
12690
12691
12692
12693
12694
12695
12696
12697
12698
12699
12700
12701
12702
12703
12704
12705
12706
12707
12708
12709
12710
12711
12712
12713
12714
12715
12716
12717
12718
12719
12720
12721
12722
12723
12724
12725
12726
12727
12728
12729
12730
12731
12732
12733
12734
12735
12736
12737
12738
12739
12740
12741
12742
12743
12744
12745
12746
12747
12748
12749
12750
12751
12752
12753
12754
12755
12756
12757
12758
12759
12760
12761
12762
12763
12764
12765
12766
12767
12768
12769
12770
12771
12772
12773
12774
12775
12776
12777
12778
12779
12780
12781
12782
12783
12784
12785
12786
12787
12788
12789
12790
12791
12792
12793
12794
12795
12796
12797
12798
12799
12800
12801
12802
12803
12804
12805
12806
12807
12808
12809
12810
12811
12812
12813
12814
12815
12816
12817
12818
12819
12820
12821
12822
12823
12824
12825
12826
12827
12828
12829
12830
12831
12832
12833
12834
12835
12836
12837
12838
12839
12840
12841
12842
12843
12844
12845
12846
12847
12848
12849
12850
12851
12852
12853
12854
12855
12856
12857
12858
12859
12860
12861
12862
12863
12864
12865
12866
12867
12868
12869
12870
12871
12872
12873
12874
12875
12876
12877
12878
12879
12880
12881
12882
12883
12884
12885
12886
12887
12888
12889
12890
12891
12892
12893
12894
12895
12896
12897
12898
12899
12900
12901
12902
12903
12904
12905
12906
12907
12908
12909
12910
12911
12912
12913
12914
12915
12916
12917
12918
12919
12920
12921
12922
12923
12924
12925
12926
12927
12928
12929
12930
12931
12932
12933
12934
12935
12936
12937
12938
12939
12940
12941
12942
12943
12944
12945
12946
12947
12948
12949
12950
12951
12952
12953
12954
12955
12956
12957
12958
12959
12960
12961
12962
12963
12964
12965
12966
12967
12968
12969
12970
12971
12972
12973
12974
12975
12976
12977
12978
12979
12980
12981
12982
12983
12984
12985
12986
12987
12988
12989
12990
12991
12992
12993
12994
12995
12996
12997
12998
12999
13000
13001
13002
13003
13004
13005
13006
13007
13008
13009
13010
13011
13012
13013
13014
13015
13016
13017
13018
13019
13020
13021
13022
13023
13024
13025
13026
13027
13028
13029
13030
13031
13032
13033
13034
13035
13036
13037
13038
13039
13040
13041
13042
13043
13044
13045
13046
13047
13048
13049
13050
13051
13052
13053
13054
13055
13056
13057
13058
13059
13060
13061
13062
13063
13064
13065
13066
13067
13068
13069
13070
13071
13072
13073
13074
13075
13076
13077
13078
13079
13080
13081
13082
13083
13084
13085
13086
13087
13088
13089
13090
13091
13092
13093
13094
13095
13096
13097
13098
13099
13100
13101
13102
13103
13104
13105
13106
13107
13108
13109
13110
13111
13112
13113
13114
13115
13116
13117
13118
13119
13120
13121
13122
13123
13124
13125
13126
13127
13128
13129
13130
13131
13132
13133
13134
13135
13136
13137
13138
13139
13140
13141
13142
13143
13144
13145
13146
13147
13148
13149
13150
13151
13152
13153
13154
13155
13156
13157
13158
13159
13160
13161
13162
13163
13164
13165
13166
13167
13168
13169
13170
13171
13172
13173
13174
13175
13176
13177
13178
13179
13180
13181
13182
13183
13184
13185
13186
13187
13188
13189
13190
13191
13192
13193
13194
13195
13196
13197
13198
13199
13200
13201
13202
13203
13204
13205
13206
13207
13208
13209
13210
13211
13212
13213
13214
13215
13216
13217
13218
13219
13220
13221
13222
13223
13224
13225
13226
13227
13228
13229
13230
13231
13232
13233
13234
13235
13236
13237
13238
13239
13240
13241
13242
13243
13244
13245
13246
13247
13248
13249
13250
13251
13252
13253
13254
13255
13256
13257
13258
13259
13260
13261
13262
13263
13264
13265
13266
13267
13268
13269
13270
13271
13272
13273
13274
13275
13276
13277
13278
13279
13280
13281
13282
13283
13284
13285
13286
13287
13288
13289
13290
13291
13292
13293
13294
13295
13296
13297
13298
13299
13300
13301
13302
13303
13304
13305
13306
13307
13308
13309
13310
13311
13312
13313
13314
13315
13316
13317
13318
13319
13320
13321
13322
13323
13324
13325
13326
13327
13328
13329
13330
13331
13332
13333
13334
13335
13336
13337
13338
13339
13340
13341
13342
13343
13344
13345
13346
13347
13348
13349
13350
13351
13352
13353
13354
13355
13356
13357
13358
13359
13360
13361
13362
13363
13364
13365
13366
13367
13368
13369
13370
13371
13372
13373
13374
13375
13376
13377
13378
13379
13380
13381
13382
13383
13384
13385
13386
13387
13388
13389
13390
13391
13392
13393
13394
13395
13396
13397
13398
13399
13400
13401
13402
13403
13404
13405
13406
13407
13408
13409
13410
13411
13412
13413
13414
13415
13416
13417
13418
13419
13420
13421
13422
13423
13424
13425
13426
13427
13428
13429
13430
13431
13432
13433
13434
13435
13436
13437
13438
13439
13440
13441
13442
13443
13444
13445
13446
13447
13448
13449
13450
13451
13452
13453
13454
13455
13456
13457
13458
13459
13460
13461
13462
13463
13464
13465
13466
13467
13468
13469
13470
13471
13472
13473
13474
13475
13476
13477
13478
13479
13480
13481
13482
13483
13484
13485
13486
13487
13488
13489
13490
13491
13492
13493
13494
13495
13496
13497
13498
13499
13500
13501
13502
13503
13504
13505
13506
13507
13508
13509
13510
13511
13512
13513
13514
13515
13516
13517
13518
13519
13520
13521
13522
13523
13524
13525
13526
13527
13528
13529
13530
13531
13532
13533
13534
13535
13536
13537
13538
13539
13540
13541
13542
13543
13544
13545
13546
13547
13548
13549
13550
13551
13552
13553
13554
13555
13556
13557
13558
13559
13560
13561
13562
13563
13564
13565
13566
13567
13568
13569
13570
13571
13572
13573
13574
13575
13576
13577
13578
13579
13580
13581
13582
13583
13584
13585
13586
13587
13588
13589
13590
13591
13592
13593
13594
13595
13596
13597
13598
13599
13600
13601
13602
13603
13604
13605
13606
13607
13608
13609
13610
13611
13612
13613
13614
13615
13616
13617
13618
13619
13620
13621
13622
13623
13624
13625
13626
13627
13628
13629
13630
13631
13632
13633
13634
13635
13636
13637
13638
13639
13640
13641
13642
13643
13644
13645
13646
13647
13648
13649
13650
13651
13652
13653
13654
13655
13656
13657
13658
13659
13660
13661
13662
13663
13664
13665
13666
13667
13668
13669
13670
13671
13672
13673
13674
13675
13676
13677
13678
13679
13680
13681
13682
13683
13684
13685
13686
13687
13688
13689
13690
13691
13692
13693
13694
13695
13696
13697
13698
13699
13700
13701
13702
13703
13704
13705
13706
13707
13708
13709
13710
13711
13712
13713
13714
13715
13716
13717
13718
13719
13720
13721
13722
13723
13724
13725
13726
13727
13728
13729
13730
13731
13732
13733
13734
13735
13736
13737
13738
13739
13740
13741
13742
13743
13744
13745
13746
13747
13748
13749
13750
13751
13752
13753
13754
13755
13756
13757
13758
13759
13760
13761
13762
13763
13764
13765
13766
13767
13768
13769
13770
13771
13772
13773
13774
13775
13776
13777
13778
13779
13780
13781
13782
13783
13784
13785
13786
13787
13788
13789
13790
13791
13792
13793
13794
13795
13796
13797
13798
13799
13800
13801
13802
13803
13804
13805
13806
13807
13808
13809
13810
13811
13812
13813
13814
13815
13816
13817
13818
13819
13820
13821
13822
13823
13824
13825
13826
13827
13828
13829
13830
13831
13832
13833
13834
13835
13836
13837
13838
13839
13840
13841
13842
13843
13844
13845
13846
13847
13848
13849
13850
13851
13852
13853
13854
13855
13856
13857
13858
13859
13860
13861
13862
13863
13864
13865
13866
13867
13868
13869
13870
13871
13872
13873
13874
13875
13876
13877
13878
13879
13880
13881
13882
13883
13884
13885
13886
13887
13888
13889
13890
13891
13892
13893
13894
13895
13896
13897
13898
13899
13900
13901
13902
13903
13904
13905
13906
13907
13908
13909
13910
13911
13912
13913
13914
13915
13916
13917
13918
13919
13920
13921
13922
13923
13924
13925
13926
13927
13928
13929
13930
13931
13932
13933
13934
13935
13936
13937
13938
13939
13940
13941
13942
13943
13944
13945
13946
13947
13948
13949
13950
13951
13952
13953
13954
13955
13956
13957
13958
13959
13960
13961
13962
13963
13964
13965
13966
13967
13968
13969
13970
13971
13972
13973
13974
13975
13976
13977
13978
13979
13980
13981
13982
13983
13984
13985
13986
13987
13988
13989
13990
13991
13992
13993
13994
13995
13996
13997
13998
13999
14000
14001
14002
14003
14004
14005
14006
14007
14008
14009
14010
14011
14012
14013
14014
14015
14016
14017
14018
14019
14020
14021
14022
14023
14024
14025
14026
14027
14028
14029
14030
14031
14032
14033
14034
14035
14036
14037
14038
14039
14040
14041
14042
14043
14044
14045
14046
14047
14048
14049
14050
14051
14052
14053
14054
14055
14056
14057
14058
14059
14060
14061
14062
14063
14064
14065
14066
14067
14068
14069
14070
14071
14072
14073
14074
14075
14076
14077
14078
14079
14080
14081
14082
14083
14084
14085
14086
14087
14088
14089
14090
14091
14092
14093
14094
14095
14096
14097
14098
14099
14100
14101
14102
14103
14104
14105
14106
14107
14108
14109
14110
14111
14112
14113
14114
14115
14116
14117
14118
14119
14120
14121
14122
14123
14124
14125
14126
14127
14128
14129
14130
14131
14132
14133
14134
14135
14136
14137
14138
14139
14140
14141
14142
14143
14144
14145
14146
14147
14148
14149
14150
14151
14152
14153
14154
14155
14156
14157
14158
14159
14160
14161
14162
14163
14164
14165
14166
14167
14168
14169
14170
14171
14172
14173
14174
14175
14176
14177
14178
14179
14180
14181
14182
14183
14184
14185
14186
14187
14188
14189
14190
14191
14192
14193
14194
14195
14196
14197
14198
14199
14200
14201
14202
14203
14204
14205
14206
14207
14208
14209
14210
14211
14212
14213
14214
14215
14216
14217
14218
14219
14220
14221
14222
14223
14224
14225
14226
14227
14228
14229
14230
14231
14232
14233
14234
14235
14236
14237
14238
14239
14240
14241
14242
14243
14244
14245
14246
14247
14248
14249
14250
14251
14252
14253
14254
14255
14256
14257
14258
14259
14260
14261
14262
14263
14264
14265
14266
14267
14268
14269
14270
14271
14272
14273
14274
14275
14276
14277
14278
14279
14280
14281
14282
14283
14284
14285
14286
14287
14288
14289
14290
14291
14292
14293
14294
14295
14296
14297
14298
14299
14300
14301
14302
14303
14304
14305
14306
14307
14308
14309
14310
14311
14312
14313
14314
14315
14316
14317
14318
14319
14320
14321
14322
14323
14324
14325
14326
14327
14328
14329
14330
14331
14332
14333
14334
14335
14336
14337
14338
14339
14340
14341
14342
14343
14344
14345
14346
14347
14348
14349
14350
14351
14352
14353
14354
14355
14356
14357
14358
14359
14360
14361
14362
14363
14364
14365
14366
14367
14368
14369
14370
14371
14372
14373
14374
14375
14376
14377
14378
14379
14380
14381
14382
14383
14384
14385
14386
14387
14388
14389
14390
14391
14392
14393
14394
14395
14396
14397
14398
14399
14400
14401
14402
14403
14404
14405
14406
14407
14408
14409
14410
14411
14412
14413
14414
14415
14416
14417
14418
14419
14420
14421
14422
14423
14424
14425
14426
14427
14428
14429
14430
14431
14432
14433
14434
14435
14436
14437
14438
14439
14440
14441
14442
14443
14444
14445
14446
14447
14448
14449
14450
14451
14452
14453
14454
14455
14456
14457
14458
14459
14460
14461
14462
14463
14464
14465
14466
14467
14468
14469
14470
14471
14472
14473
14474
14475
14476
14477
14478
14479
14480
14481
14482
14483
14484
14485
14486
14487
14488
14489
14490
14491
14492
14493
14494
14495
14496
14497
14498
14499
14500
14501
14502
14503
14504
14505
14506
14507
14508
14509
14510
14511
14512
14513
14514
14515
14516
14517
14518
14519
14520
14521
14522
14523
14524
14525
14526
14527
14528
14529
14530
14531
14532
14533
14534
14535
14536
14537
14538
14539
14540
14541
14542
14543
14544
14545
14546
14547
14548
14549
14550
14551
14552
14553
14554
14555
14556
14557
14558
14559
14560
14561
14562
14563
14564
14565
14566
14567
14568
14569
14570
14571
14572
14573
14574
14575
14576
14577
14578
14579
14580
14581
14582
14583
14584
14585
14586
14587
14588
14589
14590
14591
14592
14593
14594
14595
14596
14597
14598
14599
14600
14601
14602
14603
14604
14605
14606
14607
14608
14609
14610
14611
14612
14613
14614
14615
14616
14617
14618
14619
14620
14621
14622
14623
14624
14625
14626
14627
14628
14629
14630
14631
14632
14633
14634
14635
14636
14637
14638
14639
14640
14641
14642
14643
14644
14645
14646
14647
14648
14649
14650
14651
14652
14653
14654
14655
14656
14657
14658
14659
14660
14661
14662
14663
14664
14665
14666
14667
14668
14669
14670
14671
14672
14673
14674
14675
14676
14677
14678
14679
14680
14681
14682
14683
14684
14685
14686
14687
14688
14689
14690
14691
14692
14693
14694
14695
14696
14697
14698
14699
14700
14701
14702
14703
14704
14705
14706
14707
14708
14709
14710
14711
14712
14713
14714
14715
14716
14717
14718
14719
14720
14721
14722
14723
14724
14725
14726
14727
14728
14729
14730
14731
14732
14733
14734
14735
14736
14737
14738
14739
14740
14741
14742
14743
14744
14745
14746
14747
14748
14749
14750
14751
14752
14753
14754
14755
14756
14757
14758
14759
14760
14761
14762
14763
14764
14765
14766
14767
14768
14769
14770
14771
14772
14773
14774
14775
14776
14777
14778
14779
14780
14781
14782
14783
14784
14785
14786
14787
14788
14789
14790
14791
14792
14793
14794
14795
14796
14797
14798
14799
14800
14801
14802
14803
14804
14805
14806
14807
14808
14809
14810
14811
14812
14813
14814
14815
14816
14817
14818
14819
14820
14821
14822
14823
14824
14825
14826
14827
14828
14829
14830
14831
14832
14833
14834
14835
14836
14837
14838
14839
14840
14841
14842
14843
14844
14845
14846
14847
14848
14849
14850
14851
14852
14853
14854
14855
14856
14857
14858
14859
14860
14861
14862
14863
14864
14865
14866
14867
14868
14869
14870
14871
14872
14873
14874
14875
14876
14877
14878
14879
14880
14881
14882
14883
14884
14885
14886
14887
14888
14889
14890
14891
14892
14893
14894
14895
14896
14897
14898
14899
14900
14901
14902
14903
14904
14905
14906
14907
14908
14909
14910
14911
14912
14913
14914
14915
14916
14917
14918
14919
14920
14921
14922
14923
14924
14925
14926
14927
14928
14929
14930
14931
14932
14933
14934
14935
14936
14937
14938
14939
14940
14941
14942
14943
14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
14969
14970
14971
14972
14973
14974
14975
14976
14977
14978
14979
14980
14981
14982
14983
14984
14985
14986
14987
14988
14989
14990
14991
14992
14993
14994
14995
14996
14997
14998
14999
15000
15001
15002
15003
15004
15005
15006
15007
15008
15009
15010
15011
15012
15013
15014
15015
15016
15017
15018
15019
15020
15021
15022
15023
15024
15025
15026
15027
15028
15029
15030
15031
15032
15033
15034
15035
15036
15037
15038
15039
15040
15041
15042
15043
15044
15045
15046
15047
15048
15049
15050
15051
15052
15053
15054
15055
15056
15057
15058
15059
15060
15061
15062
15063
15064
15065
15066
15067
15068
15069
15070
15071
15072
15073
15074
15075
15076
15077
15078
15079
15080
15081
15082
15083
15084
15085
15086
15087
15088
15089
15090
15091
15092
15093
15094
15095
15096
15097
15098
15099
15100
15101
15102
15103
15104
15105
15106
15107
15108
15109
15110
15111
15112
15113
15114
15115
15116
15117
15118
15119
15120
15121
15122
15123
15124
15125
15126
15127
15128
15129
15130
15131
15132
15133
15134
15135
15136
15137
15138
15139
15140
15141
15142
15143
15144
15145
15146
15147
15148
15149
15150
15151
15152
15153
15154
15155
15156
15157
15158
15159
15160
15161
15162
15163
15164
15165
15166
15167
15168
15169
15170
15171
15172
15173
15174
15175
15176
15177
15178
15179
15180
15181
15182
15183
15184
15185
15186
15187
15188
15189
15190
15191
15192
15193
15194
15195
15196
15197
15198
15199
15200
15201
15202
15203
15204
15205
15206
15207
15208
15209
15210
15211
15212
15213
15214
15215
15216
15217
15218
15219
15220
15221
15222
15223
15224
15225
15226
15227
15228
15229
15230
15231
15232
15233
15234
15235
15236
15237
15238
15239
15240
15241
15242
15243
15244
15245
15246
15247
15248
15249
15250
15251
15252
15253
15254
15255
15256
15257
15258
15259
15260
15261
15262
15263
15264
15265
15266
15267
15268
15269
15270
15271
15272
15273
15274
15275
15276
15277
15278
15279
15280
15281
15282
15283
15284
15285
15286
15287
15288
15289
15290
15291
15292
15293
15294
15295
15296
15297
15298
15299
15300
15301
15302
15303
15304
15305
15306
15307
15308
15309
15310
15311
15312
15313
15314
15315
15316
15317
15318
15319
15320
15321
15322
15323
15324
15325
15326
15327
15328
15329
15330
15331
15332
15333
15334
15335
15336
15337
15338
15339
15340
15341
15342
15343
15344
15345
15346
15347
15348
15349
15350
15351
15352
15353
15354
15355
15356
15357
15358
15359
15360
15361
15362
15363
15364
15365
15366
15367
15368
15369
15370
15371
15372
15373
15374
15375
15376
15377
15378
15379
15380
15381
15382
15383
15384
15385
15386
15387
15388
15389
15390
15391
15392
15393
15394
15395
15396
15397
15398
15399
15400
15401
15402
15403
15404
15405
15406
15407
15408
15409
15410
15411
15412
15413
15414
15415
15416
15417
15418
15419
15420
15421
15422
15423
15424
15425
15426
15427
15428
15429
15430
15431
15432
15433
15434
15435
15436
15437
15438
15439
15440
15441
15442
15443
15444
15445
15446
15447
15448
15449
15450
15451
15452
15453
15454
15455
15456
15457
15458
15459
15460
15461
15462
15463
15464
15465
15466
15467
15468
15469
15470
15471
15472
15473
15474
15475
15476
15477
15478
15479
15480
15481
15482
15483
15484
15485
15486
15487
15488
15489
15490
15491
15492
15493
15494
15495
15496
15497
15498
15499
15500
15501
15502
15503
15504
15505
15506
15507
15508
15509
15510
15511
15512
15513
15514
15515
15516
15517
15518
15519
15520
15521
15522
15523
15524
15525
15526
15527
15528
15529
15530
15531
15532
15533
15534
15535
15536
15537
15538
15539
15540
15541
15542
15543
15544
15545
15546
15547
15548
15549
15550
15551
15552
15553
15554
15555
15556
15557
15558
15559
15560
15561
15562
15563
15564
15565
15566
15567
15568
15569
15570
15571
15572
15573
15574
15575
15576
15577
15578
15579
15580
15581
15582
15583
15584
15585
15586
15587
15588
15589
15590
15591
15592
15593
15594
15595
15596
15597
15598
15599
15600
15601
15602
15603
15604
15605
15606
15607
15608
15609
15610
15611
15612
15613
15614
15615
15616
15617
15618
15619
15620
15621
15622
15623
15624
15625
15626
15627
15628
15629
15630
15631
15632
15633
15634
15635
15636
15637
15638
15639
15640
15641
15642
15643
15644
15645
15646
15647
15648
15649
15650
15651
15652
15653
15654
15655
15656
15657
15658
15659
15660
15661
15662
15663
15664
15665
15666
15667
15668
15669
15670
15671
15672
15673
15674
15675
15676
15677
15678
15679
15680
15681
15682
15683
15684
15685
15686
15687
15688
15689
15690
15691
15692
15693
15694
15695
15696
15697
15698
15699
15700
15701
15702
15703
15704
15705
15706
15707
15708
15709
15710
15711
15712
15713
15714
15715
15716
15717
15718
15719
15720
15721
15722
15723
15724
15725
15726
15727
15728
15729
15730
15731
15732
15733
15734
15735
15736
15737
15738
15739
15740
15741
15742
15743
15744
15745
15746
15747
15748
15749
15750
15751
15752
15753
15754
15755
15756
15757
15758
15759
15760
15761
15762
15763
15764
15765
15766
15767
15768
15769
15770
15771
15772
15773
15774
15775
15776
15777
15778
15779
15780
15781
15782
15783
15784
15785
15786
15787
15788
15789
15790
15791
15792
15793
15794
15795
15796
15797
15798
15799
15800
15801
15802
15803
15804
15805
15806
15807
15808
15809
15810
15811
15812
15813
15814
15815
15816
15817
15818
15819
15820
15821
15822
15823
15824
15825
15826
15827
15828
15829
15830
15831
15832
15833
15834
15835
15836
15837
15838
15839
15840
15841
15842
15843
15844
15845
15846
15847
15848
15849
15850
15851
15852
15853
15854
15855
15856
15857
15858
15859
15860
15861
15862
15863
15864
15865
15866
15867
15868
15869
15870
15871
15872
15873
15874
15875
15876
15877
15878
15879
15880
15881
15882
15883
15884
15885
15886
15887
15888
15889
15890
15891
15892
15893
15894
15895
15896
15897
15898
15899
15900
15901
15902
15903
15904
15905
15906
15907
15908
15909
15910
15911
15912
15913
15914
15915
15916
15917
15918
15919
15920
15921
15922
15923
15924
15925
15926
15927
15928
15929
15930
15931
15932
15933
15934
15935
15936
15937
15938
15939
15940
15941
15942
15943
15944
15945
15946
15947
15948
15949
15950
15951
15952
15953
15954
15955
15956
15957
15958
15959
15960
15961
15962
15963
15964
15965
15966
15967
15968
15969
15970
15971
15972
15973
15974
15975
15976
15977
15978
15979
15980
15981
15982
15983
15984
15985
15986
15987
15988
15989
15990
15991
15992
15993
15994
15995
15996
15997
15998
15999
16000
16001
16002
16003
16004
16005
16006
16007
16008
16009
16010
16011
16012
16013
16014
16015
16016
16017
16018
16019
16020
16021
16022
16023
16024
16025
16026
16027
16028
16029
16030
16031
16032
16033
16034
16035
16036
16037
16038
16039
16040
16041
16042
16043
16044
16045
16046
16047
16048
16049
16050
16051
16052
16053
16054
16055
16056
16057
16058
16059
16060
16061
16062
16063
16064
16065
16066
16067
16068
16069
16070
16071
16072
16073
16074
16075
16076
16077
16078
16079
16080
16081
16082
16083
16084
16085
16086
16087
16088
16089
16090
16091
16092
16093
16094
16095
16096
16097
16098
16099
16100
16101
16102
16103
16104
16105
16106
16107
16108
16109
16110
16111
16112
16113
16114
16115
16116
16117
16118
16119
16120
16121
16122
16123
16124
16125
16126
16127
16128
16129
16130
16131
16132
16133
16134
16135
16136
16137
16138
16139
16140
16141
16142
16143
16144
16145
16146
16147
16148
16149
16150
16151
16152
16153
16154
16155
16156
16157
16158
16159
16160
16161
16162
16163
16164
16165
16166
16167
16168
16169
16170
16171
16172
16173
16174
16175
16176
16177
16178
16179
16180
16181
16182
16183
16184
16185
16186
16187
16188
16189
16190
16191
16192
16193
16194
16195
16196
16197
16198
16199
16200
16201
16202
16203
16204
16205
16206
16207
16208
16209
16210
16211
16212
16213
16214
16215
16216
16217
16218
16219
16220
16221
16222
16223
16224
16225
16226
16227
16228
16229
16230
16231
16232
16233
16234
16235
16236
16237
16238
16239
16240
16241
16242
16243
16244
16245
16246
16247
16248
16249
16250
16251
16252
16253
16254
16255
16256
16257
16258
16259
16260
16261
16262
16263
16264
16265
16266
16267
16268
16269
16270
16271
16272
16273
16274
16275
16276
16277
16278
16279
16280
16281
16282
16283
16284
16285
16286
16287
16288
16289
16290
16291
16292
16293
16294
16295
16296
16297
16298
16299
16300
16301
16302
16303
16304
16305
16306
16307
16308
16309
16310
16311
16312
16313
16314
16315
16316
16317
16318
16319
16320
16321
16322
16323
16324
16325
16326
16327
16328
16329
16330
16331
16332
16333
16334
16335
16336
16337
16338
16339
16340
16341
16342
16343
16344
16345
16346
16347
16348
16349
16350
16351
16352
16353
16354
16355
16356
16357
16358
16359
16360
16361
16362
16363
16364
16365
16366
16367
16368
16369
16370
16371
16372
16373
16374
16375
16376
16377
16378
16379
16380
16381
16382
16383
16384
16385
16386
16387
16388
16389
16390
16391
16392
16393
16394
16395
16396
16397
16398
16399
16400
16401
16402
16403
16404
16405
16406
16407
16408
16409
16410
16411
16412
16413
16414
16415
16416
16417
16418
16419
16420
16421
16422
16423
16424
16425
16426
16427
16428
16429
16430
16431
16432
16433
16434
16435
16436
16437
16438
16439
16440
16441
16442
16443
16444
16445
16446
16447
16448
16449
16450
16451
16452
16453
16454
16455
16456
16457
16458
16459
16460
16461
16462
16463
16464
16465
16466
16467
16468
16469
16470
16471
16472
16473
16474
16475
16476
16477
16478
16479
16480
16481
16482
16483
16484
16485
16486
16487
16488
16489
16490
16491
16492
16493
16494
16495
16496
16497
16498
16499
16500
16501
16502
16503
16504
16505
16506
16507
16508
16509
16510
16511
16512
16513
16514
16515
16516
16517
16518
16519
16520
16521
16522
16523
16524
16525
16526
16527
16528
16529
16530
16531
16532
16533
16534
16535
16536
16537
16538
16539
16540
16541
16542
16543
16544
16545
16546
16547
16548
16549
16550
16551
16552
16553
16554
16555
16556
16557
16558
16559
16560
16561
16562
16563
16564
16565
16566
16567
16568
16569
16570
16571
16572
16573
16574
16575
16576
16577
16578
16579
16580
16581
16582
16583
16584
16585
16586
16587
16588
16589
16590
16591
16592
16593
16594
16595
16596
16597
16598
16599
16600
16601
16602
16603
16604
16605
16606
16607
16608
16609
16610
16611
16612
16613
16614
16615
16616
16617
16618
16619
16620
16621
16622
16623
16624
16625
16626
16627
16628
16629
16630
16631
16632
16633
16634
16635
16636
16637
16638
16639
16640
16641
16642
16643
16644
16645
16646
16647
16648
16649
16650
16651
16652
16653
16654
16655
16656
16657
16658
16659
16660
16661
16662
16663
16664
16665
16666
16667
16668
16669
16670
16671
16672
16673
16674
16675
16676
16677
16678
16679
16680
16681
16682
16683
16684
16685
16686
16687
16688
16689
16690
16691
16692
16693
16694
16695
16696
16697
16698
16699
16700
16701
16702
16703
16704
16705
16706
16707
16708
16709
16710
16711
16712
16713
16714
16715
16716
16717
16718
16719
16720
16721
16722
16723
16724
16725
16726
16727
16728
16729
16730
16731
16732
16733
16734
16735
16736
16737
16738
16739
16740
16741
16742
16743
16744
16745
16746
16747
16748
16749
16750
16751
16752
16753
16754
16755
16756
16757
16758
16759
16760
16761
16762
16763
16764
16765
16766
16767
16768
16769
16770
16771
16772
16773
16774
16775
16776
16777
16778
16779
16780
16781
16782
16783
16784
16785
16786
16787
16788
16789
16790
16791
16792
16793
16794
16795
16796
16797
16798
16799
16800
16801
16802
16803
16804
16805
16806
16807
16808
16809
16810
16811
16812
16813
16814
16815
16816
16817
16818
16819
16820
16821
16822
16823
16824
16825
16826
16827
16828
16829
16830
16831
16832
16833
16834
16835
16836
16837
16838
16839
16840
16841
16842
16843
16844
16845
16846
16847
16848
16849
16850
16851
16852
16853
16854
16855
16856
16857
16858
16859
16860
16861
16862
16863
16864
16865
16866
16867
16868
16869
16870
16871
16872
16873
16874
16875
16876
16877
16878
16879
16880
16881
16882
16883
16884
16885
16886
16887
16888
16889
16890
16891
16892
16893
16894
16895
16896
16897
16898
16899
16900
16901
16902
16903
16904
16905
16906
16907
16908
16909
16910
16911
16912
16913
16914
16915
16916
16917
16918
16919
16920
16921
16922
16923
16924
16925
16926
16927
16928
16929
16930
16931
16932
16933
16934
16935
16936
16937
16938
16939
16940
16941
16942
16943
16944
16945
16946
16947
16948
16949
16950
16951
16952
16953
16954
16955
16956
16957
16958
16959
16960
16961
16962
16963
16964
16965
16966
16967
16968
16969
16970
16971
16972
16973
16974
16975
16976
16977
16978
16979
16980
16981
16982
16983
16984
16985
16986
16987
16988
16989
16990
16991
16992
16993
16994
16995
16996
16997
16998
16999
17000
17001
17002
17003
17004
17005
17006
17007
17008
17009
17010
17011
17012
17013
17014
17015
17016
17017
17018
17019
17020
17021
17022
17023
17024
17025
17026
17027
17028
17029
17030
17031
17032
17033
17034
17035
17036
17037
17038
17039
17040
17041
17042
17043
17044
17045
17046
17047
17048
17049
17050
17051
17052
17053
17054
17055
17056
17057
17058
17059
17060
17061
17062
17063
17064
17065
17066
17067
17068
17069
17070
17071
17072
17073
17074
17075
17076
17077
17078
17079
17080
17081
17082
17083
17084
17085
17086
17087
17088
17089
17090
17091
17092
17093
17094
17095
17096
17097
17098
17099
17100
17101
17102
17103
17104
17105
17106
17107
17108
17109
17110
17111
17112
17113
17114
17115
17116
17117
17118
17119
17120
17121
17122
17123
17124
17125
17126
17127
17128
17129
17130
17131
17132
17133
17134
17135
17136
17137
17138
17139
17140
17141
17142
17143
17144
17145
17146
17147
17148
17149
17150
17151
17152
17153
17154
17155
17156
17157
17158
17159
17160
17161
17162
17163
17164
17165
17166
17167
17168
17169
17170
17171
17172
17173
17174
17175
17176
17177
17178
17179
17180
17181
17182
17183
17184
17185
17186
17187
17188
17189
17190
17191
17192
17193
17194
17195
17196
17197
17198
17199
17200
17201
17202
17203
17204
17205
17206
17207
17208
17209
17210
17211
17212
17213
17214
17215
17216
17217
17218
17219
17220
17221
17222
17223
17224
17225
17226
17227
17228
17229
17230
17231
17232
17233
17234
17235
17236
17237
17238
17239
17240
17241
17242
17243
17244
17245
17246
17247
17248
17249
17250
17251
17252
17253
17254
17255
17256
17257
17258
17259
17260
17261
17262
17263
17264
17265
17266
17267
17268
17269
17270
17271
17272
17273
17274
17275
17276
17277
17278
17279
17280
17281
17282
17283
17284
17285
17286
17287
17288
17289
17290
17291
17292
17293
17294
17295
17296
17297
17298
17299
17300
17301
17302
17303
17304
17305
17306
17307
17308
17309
17310
17311
17312
17313
17314
17315
17316
17317
17318
17319
17320
17321
17322
17323
17324
17325
17326
17327
17328
17329
17330
17331
17332
17333
17334
17335
17336
17337
17338
17339
17340
17341
17342
17343
17344
17345
17346
17347
17348
17349
17350
17351
17352
17353
17354
17355
17356
17357
17358
17359
17360
17361
17362
17363
17364
17365
17366
17367
17368
17369
17370
17371
17372
17373
17374
17375
17376
17377
17378
17379
17380
17381
17382
17383
17384
17385
17386
17387
17388
17389
17390
17391
17392
17393
17394
17395
17396
17397
17398
17399
17400
17401
17402
17403
17404
17405
17406
17407
17408
17409
17410
17411
17412
17413
17414
17415
17416
17417
17418
17419
17420
17421
17422
17423
17424
17425
17426
17427
17428
17429
17430
17431
17432
17433
17434
17435
17436
17437
17438
17439
17440
17441
17442
17443
17444
17445
17446
17447
17448
17449
17450
17451
17452
17453
17454
17455
17456
17457
17458
17459
17460
17461
17462
17463
17464
17465
17466
17467
17468
17469
17470
17471
17472
17473
17474
17475
17476
17477
17478
17479
17480
17481
17482
17483
17484
17485
17486
17487
17488
17489
17490
17491
17492
17493
17494
17495
17496
17497
17498
17499
17500
17501
17502
17503
17504
17505
17506
17507
17508
17509
17510
17511
17512
17513
17514
17515
17516
17517
17518
17519
17520
17521
17522
17523
17524
17525
17526
17527
17528
17529
17530
17531
17532
17533
17534
17535
17536
17537
17538
17539
17540
17541
17542
17543
17544
17545
17546
17547
17548
17549
17550
17551
17552
17553
17554
17555
17556
17557
17558
17559
17560
17561
17562
17563
17564
17565
17566
17567
17568
17569
17570
17571
17572
17573
17574
17575
17576
17577
17578
17579
17580
17581
17582
17583
17584
17585
17586
17587
17588
17589
17590
17591
17592
17593
17594
17595
17596
17597
17598
17599
17600
17601
17602
17603
17604
17605
17606
17607
17608
17609
17610
17611
17612
17613
17614
17615
17616
17617
17618
17619
17620
17621
17622
17623
17624
17625
17626
17627
17628
17629
17630
17631
17632
17633
17634
17635
17636
17637
17638
17639
17640
17641
17642
17643
17644
17645
17646
17647
17648
17649
17650
17651
17652
17653
17654
17655
17656
17657
17658
17659
17660
17661
17662
17663
17664
17665
17666
17667
17668
17669
17670
17671
17672
17673
17674
17675
17676
17677
17678
17679
17680
17681
17682
17683
17684
17685
17686
17687
17688
17689
17690
17691
17692
17693
17694
17695
17696
17697
17698
17699
17700
17701
17702
17703
17704
17705
17706
17707
17708
17709
17710
17711
17712
17713
17714
17715
17716
17717
17718
17719
17720
17721
17722
17723
17724
17725
17726
17727
17728
17729
17730
17731
17732
17733
17734
17735
17736
17737
17738
17739
17740
17741
17742
17743
17744
17745
17746
17747
17748
17749
17750
17751
17752
17753
17754
17755
17756
17757
17758
17759
17760
17761
17762
17763
17764
17765
17766
17767
17768
17769
17770
17771
17772
17773
17774
17775
17776
17777
17778
17779
17780
17781
17782
17783
17784
17785
17786
17787
17788
17789
17790
17791
17792
17793
17794
17795
17796
17797
17798
17799
17800
17801
17802
17803
17804
17805
17806
17807
17808
17809
17810
17811
17812
17813
17814
17815
17816
17817
17818
17819
17820
17821
17822
17823
17824
17825
17826
17827
17828
17829
17830
17831
17832
17833
17834
17835
17836
17837
17838
17839
17840
17841
17842
17843
17844
17845
17846
17847
17848
17849
17850
17851
17852
17853
17854
17855
17856
17857
17858
17859
17860
17861
17862
17863
17864
17865
17866
17867
17868
17869
17870
17871
17872
17873
17874
17875
17876
17877
17878
17879
17880
17881
17882
17883
17884
17885
17886
17887
17888
17889
17890
17891
17892
17893
17894
17895
17896
17897
17898
17899
17900
17901
17902
17903
17904
17905
17906
17907
17908
17909
17910
17911
17912
17913
17914
17915
17916
17917
17918
17919
17920
17921
17922
17923
17924
17925
17926
17927
17928
17929
17930
17931
17932
17933
17934
17935
17936
17937
17938
17939
17940
17941
17942
17943
17944
17945
17946
17947
17948
17949
17950
17951
17952
17953
17954
17955
17956
17957
17958
17959
17960
17961
17962
17963
17964
17965
17966
17967
17968
17969
17970
17971
17972
17973
17974
17975
17976
17977
17978
17979
17980
17981
17982
17983
17984
17985
17986
17987
17988
17989
17990
17991
17992
17993
17994
17995
17996
17997
17998
17999
18000
18001
18002
18003
18004
18005
18006
18007
18008
18009
18010
18011
18012
18013
18014
18015
18016
18017
18018
18019
18020
18021
18022
18023
18024
18025
18026
18027
18028
18029
18030
18031
18032
18033
18034
18035
18036
18037
18038
18039
18040
18041
18042
18043
18044
18045
18046
18047
18048
18049
18050
18051
18052
18053
18054
18055
18056
18057
18058
18059
18060
18061
18062
18063
18064
18065
18066
18067
18068
18069
18070
18071
18072
18073
18074
18075
18076
18077
18078
18079
18080
18081
18082
18083
18084
18085
18086
18087
18088
18089
18090
18091
18092
18093
18094
18095
18096
18097
18098
18099
18100
18101
18102
18103
18104
18105
18106
18107
18108
18109
18110
18111
18112
18113
18114
18115
18116
18117
18118
18119
18120
18121
18122
18123
18124
18125
18126
18127
18128
18129
18130
18131
18132
18133
18134
18135
18136
18137
18138
18139
18140
18141
18142
18143
18144
18145
18146
18147
18148
18149
18150
18151
18152
18153
18154
18155
18156
18157
18158
18159
18160
18161
18162
18163
18164
18165
18166
18167
18168
18169
18170
18171
18172
18173
18174
18175
18176
18177
18178
18179
18180
18181
18182
18183
18184
18185
18186
18187
18188
18189
18190
18191
18192
18193
18194
18195
18196
18197
18198
18199
18200
18201
18202
18203
18204
18205
18206
18207
18208
18209
18210
18211
18212
18213
18214
18215
18216
18217
18218
18219
18220
18221
18222
18223
18224
18225
18226
18227
18228
18229
18230
18231
18232
18233
18234
18235
18236
18237
18238
18239
18240
18241
18242
18243
18244
18245
18246
18247
18248
18249
18250
18251
18252
18253
18254
18255
18256
18257
18258
18259
18260
18261
18262
18263
18264
18265
18266
18267
18268
18269
18270
18271
18272
18273
18274
18275
18276
18277
18278
18279
18280
18281
18282
18283
18284
18285
18286
18287
18288
18289
18290
18291
18292
18293
18294
18295
18296
18297
18298
18299
18300
18301
18302
18303
18304
18305
18306
18307
18308
18309
18310
18311
18312
18313
18314
18315
18316
18317
18318
18319
18320
18321
18322
18323
18324
18325
18326
18327
18328
18329
18330
18331
18332
18333
18334
18335
18336
18337
18338
18339
18340
18341
18342
18343
18344
18345
18346
18347
18348
18349
18350
18351
18352
18353
18354
18355
18356
18357
18358
18359
18360
18361
18362
18363
18364
18365
18366
18367
18368
18369
18370
18371
18372
18373
18374
18375
18376
18377
18378
18379
18380
18381
18382
18383
18384
18385
18386
18387
18388
18389
18390
18391
18392
18393
18394
18395
18396
18397
18398
18399
18400
18401
18402
18403
18404
18405
18406
18407
18408
18409
18410
18411
18412
18413
18414
18415
18416
18417
18418
18419
18420
18421
18422
18423
18424
18425
18426
18427
18428
18429
18430
18431
18432
18433
18434
18435
18436
18437
18438
18439
18440
18441
18442
18443
18444
18445
18446
18447
18448
18449
18450
18451
18452
18453
18454
18455
18456
18457
18458
18459
18460
18461
18462
18463
18464
18465
18466
18467
18468
18469
18470
18471
18472
18473
18474
18475
18476
18477
18478
18479
18480
18481
18482
18483
18484
18485
18486
18487
18488
18489
18490
18491
18492
18493
18494
18495
18496
18497
18498
18499
18500
18501
18502
18503
18504
18505
18506
18507
18508
18509
18510
18511
18512
18513
18514
18515
18516
18517
18518
18519
18520
18521
18522
18523
18524
18525
18526
18527
18528
18529
18530
18531
18532
18533
18534
18535
18536
18537
18538
18539
18540
18541
18542
18543
18544
18545
18546
18547
18548
18549
18550
18551
18552
18553
18554
18555
18556
18557
18558
18559
18560
18561
18562
18563
18564
18565
18566
18567
18568
18569
18570
18571
18572
18573
18574
18575
18576
18577
18578
18579
18580
18581
18582
18583
18584
18585
18586
18587
18588
18589
18590
18591
18592
18593
18594
18595
18596
18597
18598
18599
18600
18601
18602
18603
18604
18605
18606
18607
18608
18609
18610
18611
18612
18613
18614
18615
18616
18617
18618
18619
18620
18621
18622
18623
18624
18625
18626
18627
18628
18629
18630
18631
18632
18633
18634
18635
18636
18637
18638
18639
18640
18641
18642
18643
18644
18645
18646
18647
18648
18649
18650
18651
18652
18653
18654
18655
18656
18657
18658
18659
18660
18661
18662
18663
18664
18665
18666
18667
18668
18669
18670
18671
18672
18673
18674
18675
18676
18677
18678
18679
18680
18681
18682
18683
18684
18685
18686
18687
18688
18689
18690
18691
18692
18693
18694
18695
18696
18697
18698
18699
18700
18701
18702
18703
18704
18705
18706
18707
18708
18709
18710
18711
18712
18713
18714
18715
18716
18717
18718
18719
18720
18721
18722
18723
18724
18725
18726
18727
18728
18729
18730
18731
18732
18733
18734
18735
18736
18737
18738
18739
18740
18741
18742
18743
18744
18745
18746
18747
18748
18749
18750
18751
18752
18753
18754
18755
18756
18757
18758
18759
18760
18761
18762
18763
18764
18765
18766
18767
18768
18769
18770
18771
18772
18773
18774
18775
18776
18777
18778
18779
18780
18781
18782
18783
18784
18785
18786
18787
18788
18789
18790
18791
18792
18793
18794
18795
18796
18797
18798
18799
18800
18801
18802
18803
18804
18805
18806
18807
18808
18809
18810
18811
18812
18813
18814
18815
18816
18817
18818
18819
18820
18821
18822
18823
18824
18825
18826
18827
18828
18829
18830
18831
18832
18833
18834
18835
18836
18837
18838
18839
18840
18841
18842
18843
18844
18845
18846
18847
18848
18849
18850
18851
18852
18853
18854
18855
18856
18857
18858
18859
18860
18861
18862
18863
18864
18865
18866
18867
18868
18869
18870
18871
18872
18873
18874
18875
18876
18877
18878
18879
18880
18881
18882
18883
18884
18885
18886
18887
18888
18889
18890
18891
18892
18893
18894
18895
18896
18897
18898
18899
18900
18901
18902
18903
18904
18905
18906
18907
18908
18909
18910
18911
18912
18913
18914
18915
18916
18917
18918
18919
18920
18921
18922
18923
18924
18925
18926
18927
18928
18929
18930
18931
18932
18933
18934
18935
18936
18937
18938
18939
18940
18941
18942
18943
18944
18945
18946
18947
18948
18949
18950
18951
18952
18953
18954
18955
18956
18957
18958
18959
18960
18961
18962
18963
18964
18965
18966
18967
18968
18969
18970
18971
18972
18973
18974
18975
18976
18977
18978
18979
18980
18981
18982
18983
18984
18985
18986
18987
18988
18989
18990
18991
18992
18993
18994
18995
18996
18997
18998
18999
19000
19001
19002
19003
19004
19005
19006
19007
19008
19009
19010
19011
19012
19013
19014
19015
19016
19017
19018
19019
19020
19021
19022
19023
19024
19025
19026
19027
19028
19029
19030
19031
19032
19033
19034
19035
19036
19037
19038
19039
19040
19041
19042
19043
19044
19045
19046
19047
19048
19049
19050
19051
19052
19053
19054
19055
19056
19057
19058
19059
19060
19061
19062
19063
19064
19065
19066
19067
19068
19069
19070
19071
19072
19073
19074
19075
19076
19077
19078
19079
19080
19081
19082
19083
19084
19085
19086
19087
19088
19089
19090
19091
19092
19093
19094
19095
19096
19097
19098
19099
19100
19101
19102
19103
19104
19105
19106
19107
19108
19109
19110
19111
19112
19113
19114
19115
19116
19117
19118
19119
19120
19121
19122
19123
19124
19125
19126
19127
19128
19129
19130
19131
19132
19133
19134
19135
19136
19137
19138
19139
19140
19141
19142
19143
19144
19145
19146
19147
19148
19149
19150
19151
19152
19153
19154
19155
19156
19157
19158
19159
19160
19161
19162
19163
19164
19165
19166
19167
19168
19169
19170
19171
19172
19173
19174
19175
19176
19177
19178
19179
19180
19181
19182
19183
19184
19185
19186
19187
19188
19189
19190
19191
19192
19193
19194
19195
19196
19197
19198
19199
19200
19201
19202
19203
19204
19205
19206
19207
19208
19209
19210
19211
19212
19213
19214
19215
19216
19217
19218
19219
19220
19221
19222
19223
19224
19225
19226
19227
19228
19229
19230
19231
19232
19233
19234
19235
19236
19237
19238
19239
19240
19241
19242
19243
19244
19245
19246
19247
19248
19249
19250
19251
19252
19253
19254
19255
19256
19257
19258
19259
19260
19261
19262
19263
19264
19265
19266
19267
19268
19269
19270
19271
19272
19273
19274
19275
19276
19277
19278
19279
19280
19281
19282
19283
19284
19285
19286
19287
19288
19289
19290
19291
19292
19293
19294
19295
19296
19297
19298
19299
19300
19301
19302
19303
19304
19305
19306
19307
19308
19309
19310
19311
19312
19313
19314
19315
19316
19317
19318
19319
19320
19321
19322
19323
19324
19325
19326
19327
19328
19329
19330
19331
19332
19333
19334
19335
19336
19337
19338
19339
19340
19341
19342
19343
19344
19345
19346
19347
19348
19349
19350
19351
19352
19353
19354
19355
19356
19357
19358
19359
19360
19361
19362
19363
19364
19365
19366
19367
19368
19369
19370
19371
19372
19373
19374
19375
19376
19377
19378
19379
19380
19381
19382
19383
19384
19385
19386
19387
19388
19389
19390
19391
19392
19393
19394
19395
19396
19397
19398
19399
19400
19401
19402
19403
19404
19405
19406
19407
19408
19409
19410
19411
19412
19413
19414
19415
19416
19417
19418
19419
19420
19421
19422
19423
19424
19425
19426
19427
19428
19429
19430
19431
19432
19433
19434
19435
19436
19437
19438
19439
19440
19441
19442
19443
19444
19445
19446
19447
19448
19449
19450
19451
19452
19453
19454
19455
19456
19457
19458
19459
19460
19461
19462
19463
19464
19465
19466
19467
19468
19469
19470
19471
19472
19473
19474
19475
19476
19477
19478
19479
19480
19481
19482
19483
19484
19485
19486
19487
19488
19489
19490
19491
19492
19493
19494
19495
19496
19497
19498
19499
19500
19501
19502
19503
19504
19505
19506
19507
19508
19509
19510
19511
19512
19513
19514
19515
19516
19517
19518
19519
19520
19521
19522
19523
19524
19525
19526
19527
19528
19529
19530
19531
19532
19533
19534
19535
19536
19537
19538
19539
19540
19541
19542
19543
19544
19545
19546
19547
19548
19549
19550
19551
19552
19553
19554
19555
19556
19557
19558
19559
19560
19561
19562
19563
19564
19565
19566
19567
19568
19569
19570
19571
19572
19573
19574
19575
19576
19577
19578
19579
19580
19581
19582
19583
19584
19585
19586
19587
19588
19589
19590
19591
19592
19593
19594
19595
19596
19597
19598
19599
19600
19601
19602
19603
19604
19605
19606
19607
19608
19609
19610
19611
19612
19613
19614
19615
19616
19617
19618
19619
19620
19621
19622
19623
19624
19625
19626
19627
19628
19629
19630
19631
19632
19633
19634
19635
19636
19637
19638
19639
19640
19641
19642
19643
19644
19645
19646
19647
19648
19649
19650
19651
19652
19653
19654
19655
19656
19657
19658
19659
19660
19661
19662
19663
19664
19665
19666
19667
19668
19669
19670
19671
19672
19673
19674
19675
19676
19677
19678
19679
19680
19681
19682
19683
19684
19685
19686
19687
19688
19689
19690
19691
19692
19693
19694
19695
19696
19697
19698
19699
19700
19701
19702
19703
19704
19705
19706
19707
19708
19709
19710
19711
19712
19713
19714
19715
19716
19717
19718
19719
19720
19721
19722
19723
19724
19725
19726
19727
19728
19729
19730
19731
19732
19733
19734
19735
19736
19737
19738
19739
19740
19741
19742
19743
19744
19745
19746
19747
19748
19749
19750
19751
19752
19753
19754
19755
19756
19757
19758
19759
19760
19761
19762
19763
19764
19765
19766
19767
19768
19769
19770
19771
19772
19773
19774
19775
19776
19777
19778
19779
19780
19781
19782
19783
19784
19785
19786
19787
19788
19789
19790
19791
19792
19793
19794
19795
19796
19797
19798
19799
19800
19801
19802
19803
19804
19805
19806
19807
19808
19809
19810
19811
19812
19813
19814
19815
19816
19817
19818
19819
19820
19821
19822
19823
19824
19825
19826
19827
19828
19829
19830
19831
19832
19833
19834
19835
19836
19837
19838
19839
19840
19841
19842
19843
19844
19845
19846
19847
19848
19849
19850
19851
19852
19853
19854
19855
19856
19857
19858
19859
19860
19861
19862
19863
19864
19865
19866
19867
19868
19869
19870
19871
19872
19873
19874
19875
19876
19877
19878
19879
19880
19881
19882
19883
19884
19885
19886
19887
19888
19889
19890
19891
19892
19893
19894
19895
19896
19897
19898
19899
19900
19901
19902
19903
19904
19905
19906
19907
19908
19909
19910
19911
19912
19913
19914
19915
19916
19917
19918
19919
19920
19921
19922
19923
19924
19925
19926
19927
19928
19929
19930
19931
19932
19933
19934
19935
19936
19937
19938
19939
19940
19941
19942
19943
19944
19945
19946
19947
19948
19949
19950
19951
19952
19953
19954
19955
19956
19957
19958
19959
19960
19961
19962
19963
19964
19965
19966
19967
19968
19969
19970
19971
19972
19973
19974
19975
19976
19977
19978
19979
19980
19981
19982
19983
19984
19985
19986
19987
19988
19989
19990
19991
19992
19993
19994
19995
19996
19997
19998
19999
20000
20001
20002
20003
20004
20005
20006
20007
20008
20009
20010
20011
20012
20013
20014
20015
20016
20017
20018
20019
20020
20021
20022
20023
20024
20025
20026
20027
20028
20029
20030
20031
20032
20033
20034
20035
20036
20037
20038
20039
20040
20041
20042
20043
20044
20045
20046
20047
20048
20049
20050
20051
20052
20053
20054
20055
20056
20057
20058
20059
20060
20061
20062
20063
20064
20065
20066
20067
20068
20069
20070
20071
20072
20073
20074
20075
20076
20077
20078
20079
20080
20081
20082
20083
20084
20085
20086
20087
20088
20089
20090
20091
20092
20093
20094
20095
20096
20097
20098
20099
20100
20101
20102
20103
20104
20105
20106
20107
20108
20109
20110
20111
20112
20113
20114
20115
20116
20117
20118
20119
20120
20121
20122
20123
20124
20125
20126
20127
20128
20129
20130
20131
20132
20133
20134
20135
20136
20137
20138
20139
20140
20141
20142
20143
20144
20145
20146
20147
20148
20149
20150
20151
20152
20153
20154
20155
20156
20157
20158
20159
20160
20161
20162
20163
20164
20165
20166
20167
20168
20169
20170
20171
20172
20173
20174
20175
20176
20177
20178
20179
20180
20181
20182
20183
20184
20185
20186
20187
20188
20189
20190
20191
20192
20193
20194
20195
20196
20197
20198
20199
20200
20201
20202
20203
20204
20205
20206
20207
20208
20209
20210
20211
20212
20213
20214
20215
20216
20217
20218
20219
20220
20221
20222
20223
20224
20225
20226
20227
20228
20229
20230
20231
20232
20233
20234
20235
20236
20237
20238
20239
20240
20241
20242
20243
20244
20245
20246
20247
20248
20249
20250
20251
20252
20253
20254
20255
20256
20257
20258
20259
20260
20261
20262
20263
20264
20265
20266
20267
20268
20269
20270
20271
20272
20273
20274
20275
20276
20277
20278
20279
20280
20281
20282
20283
20284
20285
20286
20287
20288
20289
20290
20291
20292
20293
20294
20295
20296
20297
20298
20299
20300
20301
20302
20303
20304
20305
20306
20307
20308
20309
20310
20311
20312
20313
20314
20315
20316
20317
20318
20319
20320
20321
20322
20323
20324
20325
20326
20327
20328
20329
20330
20331
20332
20333
20334
20335
20336
20337
20338
20339
20340
20341
20342
20343
20344
20345
20346
20347
20348
20349
20350
20351
20352
20353
20354
20355
20356
20357
20358
20359
20360
20361
20362
20363
20364
20365
20366
20367
20368
20369
20370
20371
20372
20373
20374
20375
20376
20377
20378
20379
20380
20381
20382
20383
20384
20385
20386
20387
20388
20389
20390
20391
20392
20393
20394
20395
20396
20397
20398
20399
20400
20401
20402
20403
20404
20405
20406
20407
20408
20409
20410
20411
20412
20413
20414
20415
20416
20417
20418
20419
20420
20421
20422
20423
20424
20425
20426
20427
20428
20429
20430
20431
20432
20433
20434
20435
20436
20437
20438
20439
20440
20441
20442
20443
20444
20445
20446
20447
20448
20449
20450
20451
20452
20453
20454
20455
20456
20457
20458
20459
20460
20461
20462
20463
20464
20465
20466
20467
20468
20469
20470
20471
20472
20473
20474
20475
20476
20477
20478
20479
20480
20481
20482
20483
20484
20485
20486
20487
20488
20489
20490
20491
20492
20493
20494
20495
20496
20497
20498
20499
20500
20501
20502
20503
20504
20505
20506
20507
20508
20509
20510
20511
20512
20513
20514
20515
20516
20517
20518
20519
20520
20521
20522
20523
20524
20525
20526
20527
20528
20529
20530
20531
20532
20533
20534
20535
20536
20537
20538
20539
20540
20541
20542
20543
20544
20545
20546
20547
20548
20549
20550
20551
20552
20553
20554
20555
20556
20557
20558
20559
20560
20561
20562
20563
20564
20565
20566
20567
20568
20569
20570
20571
20572
20573
20574
20575
20576
20577
20578
20579
20580
20581
20582
20583
20584
20585
20586
20587
20588
20589
20590
20591
20592
20593
20594
20595
20596
20597
20598
20599
20600
20601
20602
20603
20604
20605
20606
20607
20608
20609
20610
20611
20612
20613
20614
20615
20616
20617
20618
20619
20620
20621
20622
20623
20624
20625
20626
20627
20628
20629
20630
20631
20632
20633
20634
20635
20636
20637
20638
20639
20640
20641
20642
20643
20644
20645
20646
20647
20648
20649
20650
20651
20652
20653
20654
20655
20656
20657
20658
20659
20660
20661
20662
20663
20664
20665
20666
20667
20668
20669
20670
20671
20672
20673
20674
20675
20676
20677
20678
20679
20680
20681
20682
20683
20684
20685
20686
20687
20688
20689
20690
20691
20692
20693
20694
20695
20696
20697
20698
20699
20700
20701
20702
20703
20704
20705
20706
20707
20708
20709
20710
20711
20712
20713
20714
20715
20716
20717
20718
20719
20720
20721
20722
20723
20724
20725
20726
20727
20728
20729
20730
20731
20732
20733
20734
20735
20736
20737
20738
20739
20740
20741
20742
20743
20744
20745
20746
20747
20748
20749
20750
20751
20752
20753
20754
20755
20756
20757
20758
20759
20760
20761
20762
20763
20764
20765
20766
20767
20768
20769
20770
20771
20772
20773
20774
20775
20776
20777
20778
20779
20780
20781
20782
20783
20784
20785
20786
20787
20788
20789
20790
20791
20792
20793
20794
20795
20796
20797
20798
20799
20800
20801
20802
20803
20804
20805
20806
20807
20808
20809
20810
20811
20812
20813
20814
20815
20816
20817
20818
20819
20820
20821
20822
20823
20824
20825
20826
20827
20828
20829
20830
20831
20832
20833
20834
20835
20836
20837
20838
20839
20840
20841
20842
20843
20844
20845
20846
20847
20848
20849
20850
20851
20852
20853
20854
20855
20856
20857
20858
20859
20860
20861
20862
20863
20864
20865
20866
20867
20868
20869
20870
20871
20872
20873
20874
20875
20876
20877
20878
20879
20880
20881
20882
20883
20884
20885
20886
20887
20888
20889
20890
20891
20892
20893
20894
20895
20896
20897
20898
20899
20900
20901
20902
20903
20904
20905
20906
20907
20908
20909
20910
20911
20912
20913
20914
20915
20916
20917
20918
20919
20920
20921
20922
20923
20924
20925
20926
20927
20928
20929
20930
20931
20932
20933
20934
20935
20936
20937
20938
20939
20940
20941
20942
20943
20944
20945
20946
20947
20948
20949
20950
20951
20952
20953
20954
20955
20956
20957
20958
20959
20960
20961
20962
20963
20964
20965
20966
20967
20968
20969
20970
20971
20972
20973
20974
20975
20976
20977
20978
20979
20980
20981
20982
20983
20984
20985
20986
20987
20988
20989
20990
20991
20992
20993
20994
20995
20996
20997
20998
20999
21000
21001
21002
21003
21004
21005
21006
21007
21008
21009
21010
21011
21012
21013
21014
21015
21016
21017
21018
21019
21020
21021
21022
21023
21024
21025
21026
21027
21028
21029
21030
21031
21032
21033
21034
21035
21036
21037
21038
21039
21040
21041
21042
21043
21044
21045
21046
21047
21048
21049
21050
21051
21052
21053
21054
21055
21056
21057
21058
21059
21060
21061
21062
21063
21064
21065
21066
21067
21068
21069
21070
21071
21072
21073
21074
21075
21076
21077
21078
21079
21080
21081
21082
21083
21084
21085
21086
21087
21088
21089
21090
21091
21092
21093
21094
21095
21096
21097
21098
21099
21100
21101
21102
21103
21104
21105
21106
21107
21108
21109
21110
21111
21112
21113
21114
21115
21116
21117
21118
21119
21120
21121
21122
21123
21124
21125
21126
21127
21128
21129
21130
21131
21132
21133
21134
21135
21136
21137
21138
21139
21140
21141
21142
21143
21144
21145
21146
21147
21148
21149
21150
21151
21152
21153
21154
21155
21156
21157
21158
21159
21160
21161
21162
21163
21164
21165
21166
21167
21168
21169
21170
21171
21172
21173
21174
21175
21176
21177
21178
21179
21180
21181
21182
21183
21184
21185
21186
21187
21188
21189
21190
21191
21192
21193
21194
21195
21196
21197
21198
21199
21200
21201
21202
21203
21204
21205
21206
21207
21208
21209
21210
21211
21212
21213
21214
21215
21216
21217
21218
21219
21220
21221
21222
21223
21224
21225
21226
21227
21228
21229
21230
21231
21232
21233
21234
21235
21236
21237
21238
21239
21240
21241
21242
21243
21244
21245
21246
21247
21248
21249
21250
21251
21252
21253
21254
21255
21256
21257
21258
21259
21260
21261
21262
21263
21264
21265
21266
21267
21268
21269
21270
21271
21272
21273
21274
21275
21276
21277
21278
21279
21280
21281
21282
21283
21284
21285
21286
21287
21288
21289
21290
21291
21292
21293
21294
21295
21296
21297
21298
21299
21300
21301
21302
21303
21304
21305
21306
21307
21308
21309
21310
21311
21312
21313
21314
21315
21316
21317
21318
21319
21320
21321
21322
21323
21324
21325
21326
21327
21328
21329
21330
21331
21332
21333
21334
21335
21336
21337
21338
21339
21340
21341
21342
21343
21344
21345
21346
21347
21348
21349
21350
21351
21352
21353
21354
21355
21356
21357
21358
21359
21360
21361
21362
21363
21364
21365
21366
21367
21368
21369
21370
21371
21372
21373
21374
21375
21376
21377
21378
21379
21380
21381
21382
21383
21384
21385
21386
21387
21388
21389
21390
21391
21392
21393
21394
21395
21396
21397
21398
21399
21400
21401
21402
21403
21404
21405
21406
21407
21408
21409
21410
21411
21412
21413
21414
21415
21416
21417
21418
21419
21420
21421
21422
21423
21424
21425
21426
21427
21428
21429
21430
21431
21432
21433
21434
21435
21436
21437
21438
21439
21440
21441
21442
21443
21444
21445
21446
21447
21448
21449
21450
21451
21452
21453
21454
21455
21456
21457
21458
21459
21460
21461
21462
21463
21464
21465
21466
21467
21468
21469
21470
21471
21472
21473
21474
21475
21476
21477
21478
21479
21480
21481
21482
21483
21484
21485
21486
21487
21488
21489
21490
21491
21492
21493
21494
21495
21496
21497
21498
21499
21500
21501
21502
21503
21504
21505
21506
21507
21508
21509
21510
21511
21512
21513
21514
21515
21516
21517
21518
21519
21520
21521
21522
21523
21524
21525
21526
21527
21528
21529
21530
21531
21532
21533
21534
21535
21536
21537
21538
21539
21540
21541
21542
21543
21544
21545
21546
21547
21548
21549
21550
21551
21552
21553
21554
21555
21556
21557
21558
21559
21560
21561
21562
21563
21564
21565
21566
21567
21568
21569
21570
21571
21572
21573
21574
21575
21576
21577
21578
21579
21580
21581
21582
21583
21584
21585
21586
21587
21588
21589
21590
21591
21592
21593
21594
21595
21596
21597
21598
21599
21600
21601
21602
21603
21604
21605
21606
21607
21608
21609
21610
21611
21612
21613
21614
21615
21616
21617
21618
21619
21620
21621
21622
21623
21624
21625
21626
21627
21628
21629
21630
21631
21632
21633
21634
21635
21636
21637
21638
21639
21640
21641
21642
21643
21644
21645
21646
21647
21648
21649
21650
21651
21652
21653
21654
21655
21656
21657
21658
21659
21660
21661
21662
21663
21664
21665
21666
21667
21668
21669
21670
21671
21672
21673
21674
21675
21676
21677
21678
21679
21680
21681
21682
21683
21684
21685
21686
21687
21688
21689
21690
21691
21692
21693
21694
21695
21696
21697
21698
21699
21700
21701
21702
21703
21704
21705
21706
21707
21708
21709
21710
21711
21712
21713
21714
21715
21716
21717
21718
21719
21720
21721
21722
21723
21724
21725
21726
21727
21728
21729
21730
21731
21732
21733
21734
21735
21736
21737
21738
21739
21740
21741
21742
21743
21744
21745
21746
21747
21748
21749
21750
21751
21752
21753
21754
21755
21756
21757
21758
21759
21760
21761
21762
21763
21764
21765
21766
21767
21768
21769
21770
21771
21772
21773
21774
21775
21776
21777
21778
21779
21780
21781
21782
21783
21784
21785
21786
21787
21788
21789
21790
21791
21792
21793
21794
21795
21796
21797
21798
21799
21800
21801
21802
21803
21804
21805
21806
21807
21808
21809
21810
21811
21812
21813
21814
21815
21816
21817
21818
21819
21820
21821
21822
21823
21824
21825
21826
21827
21828
21829
21830
21831
21832
21833
21834
21835
21836
21837
21838
21839
21840
21841
21842
21843
21844
21845
21846
21847
21848
21849
21850
21851
21852
21853
21854
21855
21856
21857
21858
21859
21860
21861
21862
21863
21864
21865
21866
21867
21868
21869
21870
21871
21872
21873
21874
21875
21876
21877
21878
21879
21880
21881
21882
21883
21884
21885
21886
21887
21888
21889
21890
21891
21892
21893
21894
21895
21896
21897
21898
21899
21900
21901
21902
21903
21904
21905
21906
21907
21908
21909
21910
21911
21912
21913
21914
21915
21916
21917
21918
21919
21920
21921
21922
21923
21924
21925
21926
21927
21928
21929
21930
21931
21932
21933
21934
21935
21936
21937
21938
21939
21940
21941
21942
21943
21944
21945
21946
21947
21948
21949
21950
21951
21952
21953
21954
21955
21956
21957
21958
21959
21960
21961
21962
21963
21964
21965
21966
21967
21968
21969
21970
21971
21972
21973
21974
21975
21976
21977
21978
21979
21980
21981
21982
21983
21984
21985
21986
21987
21988
21989
21990
21991
21992
21993
21994
21995
21996
21997
21998
21999
22000
22001
22002
22003
22004
22005
22006
22007
22008
22009
22010
22011
22012
22013
22014
22015
22016
22017
22018
22019
22020
22021
22022
22023
22024
22025
22026
22027
22028
22029
22030
22031
22032
22033
22034
22035
22036
22037
22038
22039
22040
22041
22042
22043
22044
22045
22046
22047
22048
22049
22050
22051
22052
22053
22054
22055
22056
22057
22058
22059
22060
22061
22062
22063
22064
22065
22066
22067
22068
22069
22070
22071
22072
22073
22074
22075
22076
22077
22078
22079
22080
22081
22082
22083
22084
22085
22086
22087
22088
22089
22090
22091
22092
22093
22094
22095
22096
22097
22098
22099
22100
22101
22102
22103
22104
22105
22106
22107
22108
22109
22110
22111
22112
22113
22114
22115
22116
22117
22118
22119
22120
22121
22122
22123
22124
22125
22126
22127
22128
22129
22130
22131
22132
22133
22134
22135
22136
22137
22138
22139
22140
22141
22142
22143
22144
22145
22146
22147
22148
22149
22150
22151
22152
22153
22154
22155
22156
22157
22158
22159
22160
22161
22162
22163
22164
22165
22166
22167
22168
22169
22170
22171
22172
22173
22174
22175
22176
22177
22178
22179
22180
22181
22182
22183
22184
22185
22186
22187
22188
22189
22190
22191
22192
22193
22194
22195
22196
22197
22198
22199
22200
22201
22202
22203
22204
22205
22206
22207
22208
22209
22210
22211
22212
22213
22214
22215
22216
22217
22218
22219
22220
22221
22222
22223
22224
22225
22226
22227
22228
22229
22230
22231
22232
22233
22234
22235
22236
22237
22238
22239
22240
22241
22242
22243
22244
22245
22246
22247
22248
22249
22250
22251
22252
22253
22254
22255
22256
22257
22258
22259
22260
22261
22262
22263
22264
22265
22266
22267
22268
22269
22270
22271
22272
22273
22274
22275
22276
22277
22278
22279
22280
22281
22282
22283
22284
22285
22286
22287
22288
22289
22290
22291
22292
22293
22294
22295
22296
22297
22298
22299
22300
22301
22302
22303
22304
22305
22306
22307
22308
22309
22310
22311
22312
22313
22314
22315
22316
22317
22318
22319
22320
22321
22322
22323
22324
22325
22326
22327
22328
22329
22330
22331
22332
22333
22334
22335
22336
22337
22338
22339
22340
22341
22342
22343
22344
22345
22346
22347
22348
22349
22350
22351
22352
22353
22354
22355
22356
22357
22358
22359
22360
22361
22362
22363
22364
22365
22366
22367
22368
22369
22370
22371
22372
22373
22374
22375
22376
22377
22378
22379
22380
22381
22382
22383
22384
22385
22386
22387
22388
22389
22390
22391
22392
22393
22394
22395
22396
22397
22398
22399
22400
22401
22402
22403
22404
22405
22406
22407
22408
22409
22410
22411
22412
22413
22414
22415
22416
22417
22418
22419
22420
22421
22422
22423
22424
22425
22426
22427
22428
22429
22430
22431
22432
22433
22434
22435
22436
22437
22438
22439
22440
22441
22442
22443
22444
22445
22446
22447
22448
22449
22450
22451
22452
22453
22454
22455
22456
22457
22458
22459
22460
22461
22462
22463
22464
22465
22466
22467
22468
22469
22470
22471
22472
22473
22474
22475
22476
22477
22478
22479
22480
22481
22482
22483
22484
22485
22486
22487
22488
22489
22490
22491
22492
22493
22494
22495
22496
22497
22498
22499
22500
22501
22502
22503
22504
22505
22506
22507
22508
22509
22510
22511
22512
22513
22514
22515
22516
22517
22518
22519
22520
22521
22522
22523
22524
22525
22526
22527
22528
22529
22530
22531
22532
22533
22534
22535
22536
22537
22538
22539
22540
22541
22542
22543
22544
22545
22546
22547
22548
22549
22550
22551
22552
22553
22554
22555
22556
22557
22558
22559
22560
22561
22562
22563
22564
22565
22566
22567
22568
22569
22570
22571
22572
22573
22574
22575
22576
22577
22578
22579
22580
22581
22582
22583
22584
22585
22586
22587
22588
22589
22590
22591
22592
22593
22594
22595
22596
22597
22598
22599
22600
22601
22602
22603
22604
22605
22606
22607
22608
22609
22610
22611
22612
22613
22614
22615
22616
22617
22618
22619
22620
22621
22622
22623
22624
22625
22626
22627
22628
22629
22630
22631
22632
22633
22634
22635
22636
22637
22638
22639
22640
22641
22642
22643
22644
22645
22646
22647
22648
22649
22650
22651
22652
22653
22654
22655
22656
22657
22658
22659
22660
22661
22662
22663
22664
22665
22666
22667
22668
22669
22670
22671
22672
22673
22674
22675
22676
22677
22678
22679
22680
22681
22682
22683
22684
22685
22686
22687
22688
22689
22690
22691
22692
22693
22694
22695
22696
22697
22698
22699
22700
22701
22702
22703
22704
22705
22706
22707
22708
22709
22710
22711
22712
22713
22714
22715
22716
22717
22718
22719
22720
22721
22722
22723
22724
22725
22726
22727
22728
22729
22730
22731
22732
22733
22734
22735
22736
22737
22738
22739
22740
22741
22742
22743
22744
22745
22746
22747
22748
22749
22750
22751
22752
22753
22754
22755
22756
22757
22758
22759
22760
22761
22762
22763
22764
22765
22766
22767
22768
22769
22770
22771
22772
22773
22774
22775
22776
22777
22778
22779
22780
22781
22782
22783
22784
22785
22786
22787
22788
22789
22790
22791
22792
22793
22794
22795
22796
22797
22798
22799
22800
22801
22802
22803
22804
22805
22806
22807
22808
22809
22810
22811
22812
22813
22814
22815
22816
22817
22818
22819
22820
22821
22822
22823
22824
22825
22826
22827
22828
22829
22830
22831
22832
22833
22834
22835
22836
22837
22838
22839
22840
22841
22842
22843
22844
22845
22846
22847
22848
22849
22850
22851
22852
22853
22854
22855
22856
22857
22858
22859
22860
22861
22862
22863
22864
22865
22866
22867
22868
22869
22870
22871
22872
22873
22874
22875
22876
22877
22878
22879
22880
22881
22882
22883
22884
22885
22886
22887
22888
22889
22890
22891
22892
22893
22894
22895
22896
22897
22898
22899
22900
22901
22902
22903
22904
22905
22906
22907
22908
22909
22910
22911
22912
22913
22914
22915
22916
22917
22918
22919
22920
22921
22922
22923
22924
22925
22926
22927
22928
22929
22930
22931
22932
22933
22934
22935
22936
22937
22938
22939
22940
22941
22942
22943
22944
22945
22946
22947
22948
22949
22950
22951
22952
22953
22954
22955
22956
22957
22958
22959
22960
22961
22962
22963
22964
22965
22966
22967
22968
22969
22970
22971
22972
22973
22974
22975
22976
22977
22978
22979
22980
22981
22982
22983
22984
22985
22986
22987
22988
22989
22990
22991
22992
22993
22994
22995
22996
22997
22998
22999
23000
23001
23002
23003
23004
23005
23006
23007
23008
23009
23010
23011
23012
23013
23014
23015
23016
23017
23018
23019
23020
23021
23022
23023
23024
23025
23026
23027
23028
23029
23030
23031
23032
23033
23034
23035
23036
23037
23038
23039
23040
23041
23042
23043
23044
23045
23046
23047
23048
23049
23050
23051
23052
23053
23054
23055
23056
23057
23058
23059
23060
23061
23062
23063
23064
23065
23066
23067
23068
23069
23070
23071
23072
23073
23074
23075
23076
23077
23078
23079
23080
23081
23082
23083
23084
23085
23086
23087
23088
23089
23090
23091
23092
23093
23094
23095
23096
23097
23098
23099
23100
23101
23102
23103
23104
23105
23106
23107
23108
23109
23110
23111
23112
23113
23114
23115
23116
23117
23118
23119
23120
23121
23122
23123
23124
23125
23126
23127
23128
23129
23130
23131
23132
23133
23134
23135
23136
23137
23138
23139
23140
23141
23142
23143
23144
23145
23146
23147
23148
23149
23150
23151
23152
23153
23154
23155
23156
23157
23158
23159
23160
23161
23162
23163
23164
23165
23166
23167
23168
23169
23170
23171
23172
23173
23174
23175
23176
23177
23178
23179
23180
23181
23182
23183
23184
23185
23186
23187
23188
23189
23190
23191
23192
23193
23194
23195
23196
23197
23198
23199
23200
23201
23202
23203
23204
23205
23206
23207
23208
23209
23210
23211
23212
23213
23214
23215
23216
23217
23218
23219
23220
23221
23222
23223
23224
23225
23226
23227
23228
23229
23230
23231
23232
23233
23234
23235
23236
23237
23238
23239
23240
23241
23242
23243
23244
23245
23246
23247
23248
23249
23250
23251
23252
23253
23254
23255
23256
23257
23258
23259
23260
23261
23262
23263
23264
23265
23266
23267
23268
23269
23270
23271
23272
23273
23274
23275
23276
23277
23278
23279
23280
23281
23282
23283
23284
23285
23286
23287
23288
23289
23290
23291
23292
23293
23294
23295
23296
23297
23298
23299
23300
23301
23302
23303
23304
23305
23306
23307
23308
23309
23310
23311
23312
23313
23314
23315
23316
23317
23318
23319
23320
23321
23322
23323
23324
23325
23326
23327
23328
23329
23330
23331
23332
23333
23334
23335
23336
23337
23338
23339
23340
23341
23342
23343
23344
23345
23346
23347
23348
23349
23350
23351
23352
23353
23354
23355
23356
23357
23358
23359
23360
23361
23362
23363
23364
23365
23366
23367
23368
23369
23370
23371
23372
23373
23374
23375
23376
23377
23378
23379
23380
23381
23382
23383
23384
23385
23386
23387
23388
23389
23390
23391
23392
23393
23394
23395
23396
23397
23398
23399
23400
23401
23402
23403
23404
23405
23406
23407
23408
23409
23410
23411
23412
23413
23414
23415
23416
23417
23418
23419
23420
23421
23422
23423
23424
23425
23426
23427
23428
23429
23430
23431
23432
23433
23434
23435
23436
23437
23438
23439
23440
23441
23442
23443
23444
23445
23446
23447
23448
23449
23450
23451
23452
23453
23454
23455
23456
23457
23458
23459
23460
23461
23462
23463
23464
23465
23466
23467
23468
23469
23470
23471
23472
23473
23474
23475
23476
23477
23478
23479
23480
23481
23482
23483
23484
23485
23486
23487
23488
23489
23490
23491
23492
23493
23494
23495
23496
23497
23498
23499
23500
23501
23502
23503
23504
23505
23506
23507
23508
23509
23510
23511
23512
23513
23514
23515
23516
23517
23518
23519
23520
23521
23522
23523
23524
23525
23526
23527
23528
23529
23530
23531
23532
23533
23534
23535
23536
23537
23538
23539
23540
23541
23542
23543
23544
23545
23546
23547
23548
23549
23550
23551
23552
23553
23554
23555
23556
23557
23558
23559
23560
23561
23562
23563
23564
23565
23566
23567
23568
23569
23570
23571
23572
23573
23574
23575
23576
23577
23578
23579
23580
23581
23582
23583
23584
23585
23586
23587
23588
23589
23590
23591
23592
23593
23594
23595
23596
23597
23598
23599
23600
23601
23602
23603
23604
23605
23606
23607
23608
23609
23610
23611
23612
23613
23614
23615
23616
23617
23618
23619
23620
23621
23622
23623
23624
23625
23626
23627
23628
23629
23630
23631
23632
23633
23634
23635
23636
23637
23638
23639
23640
23641
23642
23643
23644
23645
23646
23647
23648
23649
23650
23651
// SPDX-License-Identifier: GPL-2.0-only
/*
 * This is the new netlink-based wireless configuration interface.
 *
 * Copyright 2006-2010        Johannes Berg <johannes@sipsolutions.net>
 * Copyright 2013-2014  Intel Mobile Communications GmbH
 * Copyright 2015-2017        Intel Deutschland GmbH
 * Copyright (C) 2018-2026 Intel Corporation
 */

#include <linux/if.h>
#include <linux/module.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/if_ether.h>
#include <linux/ieee80211.h>
#include <linux/nl80211.h>
#include <linux/rtnetlink.h>
#include <linux/netlink.h>
#include <linux/nospec.h>
#include <linux/etherdevice.h>
#include <linux/if_vlan.h>
#include <linux/random.h>
#include <net/net_namespace.h>
#include <net/genetlink.h>
#include <net/cfg80211.h>
#include <net/sock.h>
#include <net/inet_connection_sock.h>
#include "core.h"
#include "nl80211.h"
#include "reg.h"
#include "rdev-ops.h"

static int nl80211_crypto_settings(struct cfg80211_registered_device *rdev,
                                   struct genl_info *info,
                                   struct cfg80211_crypto_settings *settings,
                                   int cipher_limit);

/* the netlink family */
static struct genl_family nl80211_fam;

/* multicast groups */
enum nl80211_multicast_groups {
        NL80211_MCGRP_CONFIG,
        NL80211_MCGRP_SCAN,
        NL80211_MCGRP_REGULATORY,
        NL80211_MCGRP_MLME,
        NL80211_MCGRP_VENDOR,
        NL80211_MCGRP_NAN,
        NL80211_MCGRP_TESTMODE /* keep last - ifdef! */
};

static const struct genl_multicast_group nl80211_mcgrps[] = {
        [NL80211_MCGRP_CONFIG] = { .name = NL80211_MULTICAST_GROUP_CONFIG },
        [NL80211_MCGRP_SCAN] = { .name = NL80211_MULTICAST_GROUP_SCAN },
        [NL80211_MCGRP_REGULATORY] = { .name = NL80211_MULTICAST_GROUP_REG },
        [NL80211_MCGRP_MLME] = { .name = NL80211_MULTICAST_GROUP_MLME },
        [NL80211_MCGRP_VENDOR] = { .name = NL80211_MULTICAST_GROUP_VENDOR },
        [NL80211_MCGRP_NAN] = { .name = NL80211_MULTICAST_GROUP_NAN },
#ifdef CONFIG_NL80211_TESTMODE
        [NL80211_MCGRP_TESTMODE] = { .name = NL80211_MULTICAST_GROUP_TESTMODE }
#endif
};

/* returns ERR_PTR values */
static struct wireless_dev *
__cfg80211_wdev_from_attrs(struct cfg80211_registered_device *rdev,
                           struct net *netns, struct nlattr **attrs)
{
        struct wireless_dev *result = NULL;
        bool have_ifidx = attrs[NL80211_ATTR_IFINDEX];
        bool have_wdev_id = attrs[NL80211_ATTR_WDEV];
        u64 wdev_id = 0;
        int wiphy_idx = -1;
        int ifidx = -1;

        if (!have_ifidx && !have_wdev_id)
                return ERR_PTR(-EINVAL);

        if (have_ifidx)
                ifidx = nla_get_u32(attrs[NL80211_ATTR_IFINDEX]);
        if (have_wdev_id) {
                wdev_id = nla_get_u64(attrs[NL80211_ATTR_WDEV]);
                wiphy_idx = wdev_id >> 32;
        }

        if (rdev) {
                struct wireless_dev *wdev;

                lockdep_assert_held(&rdev->wiphy.mtx);

                list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                        if (have_ifidx && wdev->netdev &&
                            wdev->netdev->ifindex == ifidx) {
                                result = wdev;
                                break;
                        }
                        if (have_wdev_id && wdev->identifier == (u32)wdev_id) {
                                result = wdev;
                                break;
                        }
                }

                return result ?: ERR_PTR(-ENODEV);
        }

        ASSERT_RTNL();

        for_each_rdev(rdev) {
                struct wireless_dev *wdev;

                if (wiphy_net(&rdev->wiphy) != netns)
                        continue;

                if (have_wdev_id && rdev->wiphy_idx != wiphy_idx)
                        continue;

                list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                        if (have_ifidx && wdev->netdev &&
                            wdev->netdev->ifindex == ifidx) {
                                result = wdev;
                                break;
                        }
                        if (have_wdev_id && wdev->identifier == (u32)wdev_id) {
                                result = wdev;
                                break;
                        }
                }

                if (result)
                        break;
        }

        if (result)
                return result;
        return ERR_PTR(-ENODEV);
}

static struct cfg80211_registered_device *
__cfg80211_rdev_from_attrs(struct net *netns, struct nlattr **attrs)
{
        struct cfg80211_registered_device *rdev = NULL, *tmp;
        struct net_device *netdev;

        ASSERT_RTNL();

        if (!attrs[NL80211_ATTR_WIPHY] &&
            !attrs[NL80211_ATTR_IFINDEX] &&
            !attrs[NL80211_ATTR_WDEV])
                return ERR_PTR(-EINVAL);

        if (attrs[NL80211_ATTR_WIPHY])
                rdev = cfg80211_rdev_by_wiphy_idx(
                                nla_get_u32(attrs[NL80211_ATTR_WIPHY]));

        if (attrs[NL80211_ATTR_WDEV]) {
                u64 wdev_id = nla_get_u64(attrs[NL80211_ATTR_WDEV]);
                struct wireless_dev *wdev;
                bool found = false;

                tmp = cfg80211_rdev_by_wiphy_idx(wdev_id >> 32);
                if (tmp) {
                        /* make sure wdev exists */
                        list_for_each_entry(wdev, &tmp->wiphy.wdev_list, list) {
                                if (wdev->identifier != (u32)wdev_id)
                                        continue;
                                found = true;
                                break;
                        }

                        if (!found)
                                tmp = NULL;

                        if (rdev && tmp != rdev)
                                return ERR_PTR(-EINVAL);
                        rdev = tmp;
                }
        }

        if (attrs[NL80211_ATTR_IFINDEX]) {
                int ifindex = nla_get_u32(attrs[NL80211_ATTR_IFINDEX]);

                netdev = __dev_get_by_index(netns, ifindex);
                if (netdev) {
                        if (netdev->ieee80211_ptr)
                                tmp = wiphy_to_rdev(
                                        netdev->ieee80211_ptr->wiphy);
                        else
                                tmp = NULL;

                        /* not wireless device -- return error */
                        if (!tmp)
                                return ERR_PTR(-EINVAL);

                        /* mismatch -- return error */
                        if (rdev && tmp != rdev)
                                return ERR_PTR(-EINVAL);

                        rdev = tmp;
                }
        }

        if (!rdev)
                return ERR_PTR(-ENODEV);

        if (netns != wiphy_net(&rdev->wiphy))
                return ERR_PTR(-ENODEV);

        return rdev;
}

/*
 * This function returns a pointer to the driver
 * that the genl_info item that is passed refers to.
 *
 * The result of this can be a PTR_ERR and hence must
 * be checked with IS_ERR() for errors.
 */
static struct cfg80211_registered_device *
cfg80211_get_dev_from_info(struct net *netns, struct genl_info *info)
{
        return __cfg80211_rdev_from_attrs(netns, info->attrs);
}

static int validate_beacon_head(const struct nlattr *attr,
                                struct netlink_ext_ack *extack)
{
        const u8 *data = nla_data(attr);
        unsigned int len = nla_len(attr);
        const struct element *elem;
        const struct ieee80211_mgmt *mgmt = (void *)data;
        const struct ieee80211_ext *ext;
        unsigned int fixedlen, hdrlen;
        bool s1g_bcn;

        if (len < offsetofend(typeof(*mgmt), frame_control))
                goto err;

        s1g_bcn = ieee80211_is_s1g_beacon(mgmt->frame_control);
        if (s1g_bcn) {
                ext = (struct ieee80211_ext *)mgmt;
                fixedlen =
                        offsetof(struct ieee80211_ext, u.s1g_beacon.variable) +
                        ieee80211_s1g_optional_len(ext->frame_control);
                hdrlen = offsetof(struct ieee80211_ext, u.s1g_beacon);
        } else {
                fixedlen = offsetof(struct ieee80211_mgmt,
                                    u.beacon.variable);
                hdrlen = offsetof(struct ieee80211_mgmt, u.beacon);
        }

        if (len < fixedlen)
                goto err;

        if (ieee80211_hdrlen(mgmt->frame_control) != hdrlen)
                goto err;

        data += fixedlen;
        len -= fixedlen;

        for_each_element(elem, data, len) {
                /* nothing */
        }

        if (for_each_element_completed(elem, data, len))
                return 0;

err:
        NL_SET_ERR_MSG_ATTR(extack, attr, "malformed beacon head");
        return -EINVAL;
}

static int validate_ie_attr(const struct nlattr *attr,
                            struct netlink_ext_ack *extack)
{
        const u8 *data = nla_data(attr);
        unsigned int len = nla_len(attr);
        const struct element *elem;

        for_each_element(elem, data, len) {
                /* nothing */
        }

        if (for_each_element_completed(elem, data, len))
                return 0;

        NL_SET_ERR_MSG_ATTR(extack, attr, "malformed information elements");
        return -EINVAL;
}

static int validate_he_capa(const struct nlattr *attr,
                            struct netlink_ext_ack *extack)
{
        if (!ieee80211_he_capa_size_ok(nla_data(attr), nla_len(attr)))
                return -EINVAL;

        return 0;
}

static int validate_supported_selectors(const struct nlattr *attr,
                                        struct netlink_ext_ack *extack)
{
        const u8 *supported_selectors = nla_data(attr);
        u8 supported_selectors_len = nla_len(attr);

        /* The top bit must not be set as it is not part of the selector */
        for (int i = 0; i < supported_selectors_len; i++) {
                if (supported_selectors[i] & 0x80)
                        return -EINVAL;
        }

        return 0;
}

static int validate_nan_cluster_id(const struct nlattr *attr,
                                   struct netlink_ext_ack *extack)
{
        const u8 *data = nla_data(attr);
        unsigned int len = nla_len(attr);
        static const u8 cluster_id_prefix[4] = {0x50, 0x6f, 0x9a, 0x1};

        if (len != ETH_ALEN) {
                NL_SET_ERR_MSG_ATTR(extack, attr, "bad cluster id length");
                return -EINVAL;
        }

        if (memcmp(data, cluster_id_prefix, sizeof(cluster_id_prefix))) {
                NL_SET_ERR_MSG_ATTR(extack, attr, "invalid cluster id prefix");
                return -EINVAL;
        }

        return 0;
}

static int validate_nan_avail_blob(const struct nlattr *attr,
                                   struct netlink_ext_ack *extack)
{
        const u8 *data = nla_data(attr);
        unsigned int len = nla_len(attr);
        u16 attr_len;

        /* Need at least: Attr ID (1) + Length (2) */
        if (len < 3) {
                NL_SET_ERR_MSG_FMT(extack,
                                   "NAN Availability: Too short (need at least 3 bytes, have %u)",
                                   len);
                return -EINVAL;
        }

        if (data[0] != 0x12) {
                NL_SET_ERR_MSG_FMT(extack,
                                   "NAN Availability: Invalid Attribute ID 0x%02x (expected 0x12)",
                                   data[0]);
                return -EINVAL;
        }

        attr_len = get_unaligned_le16(&data[1]);

        if (attr_len != len - 3) {
                NL_SET_ERR_MSG_FMT(extack,
                                   "NAN Availability: Length field (%u) doesn't match data length (%u)",
                                   attr_len, len - 3);
                return -EINVAL;
        }

        return 0;
}

static int validate_nan_ulw(const struct nlattr *attr,
                            struct netlink_ext_ack *extack)
{
        const u8 *data = nla_data(attr);
        unsigned int len = nla_len(attr);
        unsigned int pos = 0;

        while (pos < len) {
                u16 attr_len;

                /* Need at least: Attr ID (1) + Length (2) */
                if (pos + 3 > len) {
                        NL_SET_ERR_MSG_FMT(extack,
                                           "ULW: Incomplete header (need 3 bytes, have %u)",
                                           len - pos);
                        return -EINVAL;
                }

                if (data[pos] != 0x17) {
                        NL_SET_ERR_MSG_FMT(extack,
                                           "ULW: Invalid Attribute ID 0x%02x (expected 0x17)",
                                           data[pos]);
                        return -EINVAL;
                }
                pos++;

                /* Length is in little-endian format */
                attr_len = get_unaligned_le16(&data[pos]);
                pos += 2;

                /*
                 * Check if length is one of the valid values: 16 (no
                 * channel/band entry included), 18 (band entry included),
                 * 21 (channel entry included without Auxiliary channel bitmap),
                 * or 23 (channel entry included with Auxiliary channel bitmap).
                 */
                if (attr_len != 16 && attr_len != 18 && attr_len != 21 &&
                    attr_len != 23) {
                        NL_SET_ERR_MSG_FMT(extack,
                                           "ULW: Invalid length %u (must be 16, 18, 21, or 23)",
                                           attr_len);
                        return -EINVAL;
                }

                if (pos + attr_len > len) {
                        NL_SET_ERR_MSG_FMT(extack,
                                           "ULW: Length field (%u) exceeds remaining data (%u)",
                                           attr_len, len - pos);
                        return -EINVAL;
                }

                pos += attr_len;
        }

        return 0;
}

static int validate_uhr_capa(const struct nlattr *attr,
                             struct netlink_ext_ack *extack)
{
        const u8 *data = nla_data(attr);
        unsigned int len = nla_len(attr);

        if (!ieee80211_uhr_capa_size_ok(data, len, false))
                return -EINVAL;
        return 0;
}

static int validate_uhr_operation(const struct nlattr *attr,
                                  struct netlink_ext_ack *extack)
{
        const u8 *data = nla_data(attr);
        unsigned int len = nla_len(attr);

        if (!ieee80211_uhr_oper_size_ok(data, len))
                return -EINVAL;
        return 0;
}

/* policy for the attributes */
static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR];

static const struct nla_policy
nl80211_ftm_responder_policy[NL80211_FTM_RESP_ATTR_MAX + 1] = {
        [NL80211_FTM_RESP_ATTR_ENABLED] = { .type = NLA_FLAG, },
        [NL80211_FTM_RESP_ATTR_LCI] = { .type = NLA_BINARY,
                                        .len = U8_MAX },
        [NL80211_FTM_RESP_ATTR_CIVICLOC] = { .type = NLA_BINARY,
                                             .len = U8_MAX },
};

static const struct nla_policy
nl80211_pmsr_ftm_req_attr_policy[NL80211_PMSR_FTM_REQ_ATTR_MAX + 1] = {
        [NL80211_PMSR_FTM_REQ_ATTR_ASAP] = { .type = NLA_FLAG },
        [NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE] =
                NLA_POLICY_RANGE(NLA_U32, NL80211_PREAMBLE_LEGACY,
                                 NL80211_PREAMBLE_HE),
        [NL80211_PMSR_FTM_REQ_ATTR_NUM_BURSTS_EXP] =
                NLA_POLICY_MAX(NLA_U8, 15),
        [NL80211_PMSR_FTM_REQ_ATTR_BURST_PERIOD] = { .type = NLA_U16 },
        [NL80211_PMSR_FTM_REQ_ATTR_BURST_DURATION] =
                NLA_POLICY_MAX(NLA_U8, 15),
        [NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST] = { .type = NLA_U8 },
        [NL80211_PMSR_FTM_REQ_ATTR_NUM_FTMR_RETRIES] = { .type = NLA_U8 },
        [NL80211_PMSR_FTM_REQ_ATTR_REQUEST_LCI] = { .type = NLA_FLAG },
        [NL80211_PMSR_FTM_REQ_ATTR_REQUEST_CIVICLOC] = { .type = NLA_FLAG },
        [NL80211_PMSR_FTM_REQ_ATTR_TRIGGER_BASED] = { .type = NLA_FLAG },
        [NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED] = { .type = NLA_FLAG },
        [NL80211_PMSR_FTM_REQ_ATTR_LMR_FEEDBACK] = { .type = NLA_FLAG },
        [NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR] = { .type = NLA_U8 },
        [NL80211_PMSR_FTM_REQ_ATTR_RSTA] = { .type = NLA_FLAG },
        [NL80211_PMSR_FTM_REQ_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS] = {
                .type = NLA_U32
        },
        [NL80211_PMSR_FTM_REQ_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS] = {
                .type = NLA_U32
        },
        [NL80211_PMSR_FTM_REQ_ATTR_NOMINAL_TIME] = { .type = NLA_U32 },
        [NL80211_PMSR_FTM_REQ_ATTR_AW_DURATION] = NLA_POLICY_MAX(NLA_U32, 255),
        [NL80211_PMSR_FTM_REQ_ATTR_NUM_MEASUREMENTS] = { .type = NLA_U32 },
        [NL80211_PMSR_FTM_REQ_ATTR_INGRESS] = { .type = NLA_U64 },
        [NL80211_PMSR_FTM_REQ_ATTR_EGRESS] = { .type = NLA_U64 },
        [NL80211_PMSR_FTM_REQ_ATTR_PD_SUPPRESS_RESULTS] = { .type = NLA_FLAG },
};

static const struct nla_policy
nl80211_pmsr_req_data_policy[NL80211_PMSR_TYPE_MAX + 1] = {
        [NL80211_PMSR_TYPE_FTM] =
                NLA_POLICY_NESTED(nl80211_pmsr_ftm_req_attr_policy),
};

static const struct nla_policy
nl80211_pmsr_req_attr_policy[NL80211_PMSR_REQ_ATTR_MAX + 1] = {
        [NL80211_PMSR_REQ_ATTR_DATA] =
                NLA_POLICY_NESTED(nl80211_pmsr_req_data_policy),
        [NL80211_PMSR_REQ_ATTR_GET_AP_TSF] = { .type = NLA_FLAG },
};

static const struct nla_policy
nl80211_pmsr_peer_attr_policy[NL80211_PMSR_PEER_ATTR_MAX + 1] = {
        [NL80211_PMSR_PEER_ATTR_ADDR] = NLA_POLICY_ETH_ADDR,
        [NL80211_PMSR_PEER_ATTR_CHAN] = NLA_POLICY_NESTED(nl80211_policy),
        [NL80211_PMSR_PEER_ATTR_REQ] =
                NLA_POLICY_NESTED(nl80211_pmsr_req_attr_policy),
        [NL80211_PMSR_PEER_ATTR_RESP] = { .type = NLA_REJECT },
        [NL80211_PMSR_PEER_ATTR_REQ_TYPE] =
                NLA_POLICY_MAX(NLA_U32, NL80211_PMSR_FTM_REQ_TYPE_MAX),
};

static const struct nla_policy
nl80211_pmsr_attr_policy[NL80211_PMSR_ATTR_MAX + 1] = {
        [NL80211_PMSR_ATTR_MAX_PEERS] = { .type = NLA_REJECT },
        [NL80211_PMSR_ATTR_REPORT_AP_TSF] = { .type = NLA_REJECT },
        [NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR] = { .type = NLA_REJECT },
        [NL80211_PMSR_ATTR_TYPE_CAPA] = { .type = NLA_REJECT },
        [NL80211_PMSR_ATTR_PEERS] =
                NLA_POLICY_NESTED_ARRAY(nl80211_pmsr_peer_attr_policy),
};

static const struct nla_policy
he_obss_pd_policy[NL80211_HE_OBSS_PD_ATTR_MAX + 1] = {
        [NL80211_HE_OBSS_PD_ATTR_MIN_OFFSET] =
                NLA_POLICY_RANGE(NLA_U8, 1, 20),
        [NL80211_HE_OBSS_PD_ATTR_MAX_OFFSET] =
                NLA_POLICY_RANGE(NLA_U8, 1, 20),
        [NL80211_HE_OBSS_PD_ATTR_NON_SRG_MAX_OFFSET] =
                NLA_POLICY_RANGE(NLA_U8, 1, 20),
        [NL80211_HE_OBSS_PD_ATTR_BSS_COLOR_BITMAP] =
                NLA_POLICY_EXACT_LEN(8),
        [NL80211_HE_OBSS_PD_ATTR_PARTIAL_BSSID_BITMAP] =
                NLA_POLICY_EXACT_LEN(8),
        [NL80211_HE_OBSS_PD_ATTR_SR_CTRL] = { .type = NLA_U8 },
};

static const struct nla_policy
he_bss_color_policy[NL80211_HE_BSS_COLOR_ATTR_MAX + 1] = {
        [NL80211_HE_BSS_COLOR_ATTR_COLOR] = NLA_POLICY_RANGE(NLA_U8, 1, 63),
        [NL80211_HE_BSS_COLOR_ATTR_DISABLED] = { .type = NLA_FLAG },
        [NL80211_HE_BSS_COLOR_ATTR_PARTIAL] = { .type = NLA_FLAG },
};

static const struct nla_policy nl80211_txattr_policy[NL80211_TXRATE_MAX + 1] = {
        [NL80211_TXRATE_LEGACY] = { .type = NLA_BINARY,
                                    .len = NL80211_MAX_SUPP_RATES },
        [NL80211_TXRATE_HT] = { .type = NLA_BINARY,
                                .len = NL80211_MAX_SUPP_HT_RATES },
        [NL80211_TXRATE_VHT] = NLA_POLICY_EXACT_LEN_WARN(sizeof(struct nl80211_txrate_vht)),
        [NL80211_TXRATE_GI] = { .type = NLA_U8 },
        [NL80211_TXRATE_HE] = NLA_POLICY_EXACT_LEN(sizeof(struct nl80211_txrate_he)),
        [NL80211_TXRATE_HE_GI] =  NLA_POLICY_RANGE(NLA_U8,
                                                   NL80211_RATE_INFO_HE_GI_0_8,
                                                   NL80211_RATE_INFO_HE_GI_3_2),
        [NL80211_TXRATE_HE_LTF] = NLA_POLICY_RANGE(NLA_U8,
                                                   NL80211_RATE_INFO_HE_1XLTF,
                                                   NL80211_RATE_INFO_HE_4XLTF),
        [NL80211_TXRATE_EHT] = NLA_POLICY_EXACT_LEN(sizeof(struct nl80211_txrate_eht)),
        [NL80211_TXRATE_EHT_GI] =  NLA_POLICY_RANGE(NLA_U8,
                                                   NL80211_RATE_INFO_EHT_GI_0_8,
                                                   NL80211_RATE_INFO_EHT_GI_3_2),
        [NL80211_TXRATE_EHT_LTF] = NLA_POLICY_RANGE(NLA_U8,
                                                   NL80211_RATE_INFO_EHT_1XLTF,
                                                   NL80211_RATE_INFO_EHT_8XLTF),

};

static const struct nla_policy
nl80211_tid_config_attr_policy[NL80211_TID_CONFIG_ATTR_MAX + 1] = {
        [NL80211_TID_CONFIG_ATTR_VIF_SUPP] = { .type = NLA_U64 },
        [NL80211_TID_CONFIG_ATTR_PEER_SUPP] = { .type = NLA_U64 },
        [NL80211_TID_CONFIG_ATTR_OVERRIDE] = { .type = NLA_FLAG },
        [NL80211_TID_CONFIG_ATTR_TIDS] = NLA_POLICY_RANGE(NLA_U16, 1, 0xff),
        [NL80211_TID_CONFIG_ATTR_NOACK] =
                        NLA_POLICY_MAX(NLA_U8, NL80211_TID_CONFIG_DISABLE),
        [NL80211_TID_CONFIG_ATTR_RETRY_SHORT] = NLA_POLICY_MIN(NLA_U8, 1),
        [NL80211_TID_CONFIG_ATTR_RETRY_LONG] = NLA_POLICY_MIN(NLA_U8, 1),
        [NL80211_TID_CONFIG_ATTR_AMPDU_CTRL] =
                        NLA_POLICY_MAX(NLA_U8, NL80211_TID_CONFIG_DISABLE),
        [NL80211_TID_CONFIG_ATTR_RTSCTS_CTRL] =
                        NLA_POLICY_MAX(NLA_U8, NL80211_TID_CONFIG_DISABLE),
        [NL80211_TID_CONFIG_ATTR_AMSDU_CTRL] =
                        NLA_POLICY_MAX(NLA_U8, NL80211_TID_CONFIG_DISABLE),
        [NL80211_TID_CONFIG_ATTR_TX_RATE_TYPE] =
                        NLA_POLICY_MAX(NLA_U8, NL80211_TX_RATE_FIXED),
        [NL80211_TID_CONFIG_ATTR_TX_RATE] =
                        NLA_POLICY_NESTED(nl80211_txattr_policy),
};

static const struct nla_policy
nl80211_fils_discovery_policy[NL80211_FILS_DISCOVERY_ATTR_MAX + 1] = {
        [NL80211_FILS_DISCOVERY_ATTR_INT_MIN] = NLA_POLICY_MAX(NLA_U32, 10000),
        [NL80211_FILS_DISCOVERY_ATTR_INT_MAX] = NLA_POLICY_MAX(NLA_U32, 10000),
        [NL80211_FILS_DISCOVERY_ATTR_TMPL] =
                        NLA_POLICY_RANGE(NLA_BINARY,
                                         NL80211_FILS_DISCOVERY_TMPL_MIN_LEN,
                                         IEEE80211_MAX_DATA_LEN),
};

static const struct nla_policy
nl80211_unsol_bcast_probe_resp_policy[NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_MAX + 1] = {
        [NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_INT] = NLA_POLICY_MAX(NLA_U32, 20),
        [NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_TMPL] = { .type = NLA_BINARY,
                                                       .len = IEEE80211_MAX_DATA_LEN }
};

static const struct nla_policy
sar_specs_policy[NL80211_SAR_ATTR_SPECS_MAX + 1] = {
        [NL80211_SAR_ATTR_SPECS_POWER] = { .type = NLA_S32 },
        [NL80211_SAR_ATTR_SPECS_RANGE_INDEX] = {.type = NLA_U32 },
};

static const struct nla_policy
sar_policy[NL80211_SAR_ATTR_MAX + 1] = {
        [NL80211_SAR_ATTR_TYPE] = NLA_POLICY_MAX(NLA_U32, NUM_NL80211_SAR_TYPE),
        [NL80211_SAR_ATTR_SPECS] = NLA_POLICY_NESTED_ARRAY(sar_specs_policy),
};

static const struct nla_policy
nl80211_mbssid_config_policy[NL80211_MBSSID_CONFIG_ATTR_MAX + 1] = {
        [NL80211_MBSSID_CONFIG_ATTR_MAX_INTERFACES] = NLA_POLICY_MIN(NLA_U8, 2),
        [NL80211_MBSSID_CONFIG_ATTR_MAX_EMA_PROFILE_PERIODICITY] =
                                                NLA_POLICY_MIN(NLA_U8, 1),
        [NL80211_MBSSID_CONFIG_ATTR_INDEX] = { .type = NLA_U8 },
        [NL80211_MBSSID_CONFIG_ATTR_TX_IFINDEX] = { .type = NLA_U32 },
        [NL80211_MBSSID_CONFIG_ATTR_EMA] = { .type = NLA_FLAG },
        [NL80211_MBSSID_CONFIG_ATTR_TX_LINK_ID] =
                NLA_POLICY_RANGE(NLA_U8, 0, IEEE80211_MLD_MAX_NUM_LINKS - 1),
};

static const struct nla_policy
nl80211_sta_wme_policy[NL80211_STA_WME_MAX + 1] = {
        [NL80211_STA_WME_UAPSD_QUEUES] = { .type = NLA_U8 },
        [NL80211_STA_WME_MAX_SP] = { .type = NLA_U8 },
};

static const struct nla_policy
nl80211_s1g_short_beacon[NL80211_S1G_SHORT_BEACON_ATTR_MAX + 1] = {
        [NL80211_S1G_SHORT_BEACON_ATTR_HEAD] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_beacon_head,
                                       IEEE80211_MAX_DATA_LEN),
        [NL80211_S1G_SHORT_BEACON_ATTR_TAIL] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_ie_attr,
                                       IEEE80211_MAX_DATA_LEN),
};

static const struct nla_policy
nl80211_nan_band_conf_policy[NL80211_NAN_BAND_CONF_ATTR_MAX + 1] = {
        [NL80211_NAN_BAND_CONF_BAND] = NLA_POLICY_MAX(NLA_U8,
                                                      NUM_NL80211_BANDS - 1),
        [NL80211_NAN_BAND_CONF_FREQ] = { .type = NLA_U16 },
        [NL80211_NAN_BAND_CONF_RSSI_CLOSE] = NLA_POLICY_MIN(NLA_S8, -59),
        [NL80211_NAN_BAND_CONF_RSSI_MIDDLE] = NLA_POLICY_MIN(NLA_S8, -74),
        [NL80211_NAN_BAND_CONF_WAKE_DW] = NLA_POLICY_MAX(NLA_U8, 5),
        [NL80211_NAN_BAND_CONF_DISABLE_SCAN] = { .type = NLA_FLAG },
};

static const struct nla_policy
nl80211_nan_peer_map_policy[NL80211_NAN_PEER_MAP_ATTR_MAX + 1] = {
        [NL80211_NAN_PEER_MAP_ATTR_MAP_ID] = NLA_POLICY_MAX(NLA_U8, 15),
        [NL80211_NAN_PEER_MAP_ATTR_TIME_SLOTS] =
                NLA_POLICY_EXACT_LEN(CFG80211_NAN_SCHED_NUM_TIME_SLOTS),
};

static const struct nla_policy
nl80211_nan_conf_policy[NL80211_NAN_CONF_ATTR_MAX + 1] = {
        [NL80211_NAN_CONF_CLUSTER_ID] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_nan_cluster_id,
                                       ETH_ALEN),
        [NL80211_NAN_CONF_EXTRA_ATTRS] = { .type = NLA_BINARY,
                                           .len = IEEE80211_MAX_DATA_LEN},
        [NL80211_NAN_CONF_VENDOR_ELEMS] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_ie_attr,
                                       IEEE80211_MAX_DATA_LEN),
        [NL80211_NAN_CONF_BAND_CONFIGS] =
                NLA_POLICY_NESTED_ARRAY(nl80211_nan_band_conf_policy),
        [NL80211_NAN_CONF_SCAN_PERIOD] = { .type = NLA_U16 },
        [NL80211_NAN_CONF_SCAN_DWELL_TIME] = NLA_POLICY_RANGE(NLA_U16, 50, 512),
        [NL80211_NAN_CONF_DISCOVERY_BEACON_INTERVAL] =
                NLA_POLICY_RANGE(NLA_U8, 50, 200),
        [NL80211_NAN_CONF_NOTIFY_DW] = { .type = NLA_FLAG },
};

static const struct netlink_range_validation nl80211_punct_bitmap_range = {
        .min = 0,
        .max = 0xffff,
};

static const struct netlink_range_validation q_range = {
        .max = INT_MAX,
};

static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = {
        [0] = { .strict_start_type = NL80211_ATTR_HE_OBSS_PD },
        [NL80211_ATTR_WIPHY] = { .type = NLA_U32 },
        [NL80211_ATTR_WIPHY_NAME] = { .type = NLA_NUL_STRING,
                                      .len = 20-1 },
        [NL80211_ATTR_WIPHY_TXQ_PARAMS] = { .type = NLA_NESTED },

        [NL80211_ATTR_WIPHY_FREQ] = { .type = NLA_U32 },
        [NL80211_ATTR_WIPHY_CHANNEL_TYPE] = { .type = NLA_U32 },
        [NL80211_ATTR_WIPHY_EDMG_CHANNELS] = NLA_POLICY_RANGE(NLA_U8,
                                                NL80211_EDMG_CHANNELS_MIN,
                                                NL80211_EDMG_CHANNELS_MAX),
        [NL80211_ATTR_WIPHY_EDMG_BW_CONFIG] = NLA_POLICY_RANGE(NLA_U8,
                                                NL80211_EDMG_BW_CONFIG_MIN,
                                                NL80211_EDMG_BW_CONFIG_MAX),

        [NL80211_ATTR_CHANNEL_WIDTH] = { .type = NLA_U32 },
        [NL80211_ATTR_CENTER_FREQ1] = { .type = NLA_U32 },
        [NL80211_ATTR_CENTER_FREQ1_OFFSET] = NLA_POLICY_RANGE(NLA_U32, 0, 999),
        [NL80211_ATTR_CENTER_FREQ2] = { .type = NLA_U32 },

        [NL80211_ATTR_WIPHY_RETRY_SHORT] = NLA_POLICY_MIN(NLA_U8, 1),
        [NL80211_ATTR_WIPHY_RETRY_LONG] = NLA_POLICY_MIN(NLA_U8, 1),
        [NL80211_ATTR_WIPHY_FRAG_THRESHOLD] = { .type = NLA_U32 },
        [NL80211_ATTR_WIPHY_RTS_THRESHOLD] = { .type = NLA_U32 },
        [NL80211_ATTR_WIPHY_COVERAGE_CLASS] = { .type = NLA_U8 },
        [NL80211_ATTR_WIPHY_DYN_ACK] = { .type = NLA_FLAG },

        [NL80211_ATTR_IFTYPE] = NLA_POLICY_MAX(NLA_U32, NL80211_IFTYPE_MAX),
        [NL80211_ATTR_IFINDEX] = { .type = NLA_U32 },
        [NL80211_ATTR_IFNAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ-1 },

        [NL80211_ATTR_MAC] = NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN),
        [NL80211_ATTR_PREV_BSSID] = NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN),

        [NL80211_ATTR_KEY] = { .type = NLA_NESTED, },
        [NL80211_ATTR_KEY_DATA] = { .type = NLA_BINARY,
                                    .len = WLAN_MAX_KEY_LEN },
        [NL80211_ATTR_KEY_IDX] = NLA_POLICY_MAX(NLA_U8, 7),
        [NL80211_ATTR_KEY_CIPHER] = { .type = NLA_U32 },
        [NL80211_ATTR_KEY_DEFAULT] = { .type = NLA_FLAG },
        [NL80211_ATTR_KEY_SEQ] = { .type = NLA_BINARY, .len = 16 },
        [NL80211_ATTR_KEY_TYPE] =
                NLA_POLICY_MAX(NLA_U32, NUM_NL80211_KEYTYPES),

        [NL80211_ATTR_BEACON_INTERVAL] = { .type = NLA_U32 },
        [NL80211_ATTR_DTIM_PERIOD] = { .type = NLA_U32 },
        [NL80211_ATTR_BEACON_HEAD] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_beacon_head,
                                       IEEE80211_MAX_DATA_LEN),
        [NL80211_ATTR_BEACON_TAIL] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_ie_attr,
                                       IEEE80211_MAX_DATA_LEN),
        [NL80211_ATTR_STA_AID] =
                NLA_POLICY_RANGE(NLA_U16, 1, IEEE80211_MAX_AID),
        [NL80211_ATTR_STA_FLAGS] = { .type = NLA_NESTED },
        [NL80211_ATTR_STA_LISTEN_INTERVAL] = { .type = NLA_U16 },
        [NL80211_ATTR_STA_SUPPORTED_RATES] = { .type = NLA_BINARY,
                                               .len = NL80211_MAX_SUPP_RATES },
        [NL80211_ATTR_STA_PLINK_ACTION] =
                NLA_POLICY_MAX(NLA_U8, NUM_NL80211_PLINK_ACTIONS - 1),
        [NL80211_ATTR_STA_TX_POWER_SETTING] =
                NLA_POLICY_RANGE(NLA_U8,
                                 NL80211_TX_POWER_AUTOMATIC,
                                 NL80211_TX_POWER_FIXED),
        [NL80211_ATTR_STA_TX_POWER] = { .type = NLA_S16 },
        [NL80211_ATTR_STA_VLAN] = { .type = NLA_U32 },
        [NL80211_ATTR_MNTR_FLAGS] = { /* NLA_NESTED can't be empty */ },
        [NL80211_ATTR_MESH_ID] = { .type = NLA_BINARY,
                                   .len = IEEE80211_MAX_MESH_ID_LEN },
        [NL80211_ATTR_MPATH_NEXT_HOP] = NLA_POLICY_ETH_ADDR_COMPAT,

        /* allow 3 for NUL-termination, we used to declare this NLA_STRING */
        [NL80211_ATTR_REG_ALPHA2] = NLA_POLICY_RANGE(NLA_BINARY, 2, 3),
        [NL80211_ATTR_REG_RULES] = { .type = NLA_NESTED },

        [NL80211_ATTR_BSS_CTS_PROT] = { .type = NLA_U8 },
        [NL80211_ATTR_BSS_SHORT_PREAMBLE] = { .type = NLA_U8 },
        [NL80211_ATTR_BSS_SHORT_SLOT_TIME] = { .type = NLA_U8 },
        [NL80211_ATTR_BSS_BASIC_RATES] = { .type = NLA_BINARY,
                                           .len = NL80211_MAX_SUPP_RATES },
        [NL80211_ATTR_BSS_HT_OPMODE] = { .type = NLA_U16 },

        [NL80211_ATTR_MESH_CONFIG] = { .type = NLA_NESTED },
        [NL80211_ATTR_SUPPORT_MESH_AUTH] = { .type = NLA_FLAG },

        [NL80211_ATTR_HT_CAPABILITY] = NLA_POLICY_EXACT_LEN_WARN(NL80211_HT_CAPABILITY_LEN),

        [NL80211_ATTR_MGMT_SUBTYPE] = { .type = NLA_U8 },
        [NL80211_ATTR_IE] = NLA_POLICY_VALIDATE_FN(NLA_BINARY,
                                                   validate_ie_attr,
                                                   IEEE80211_MAX_DATA_LEN),
        [NL80211_ATTR_SCAN_FREQUENCIES] = { .type = NLA_NESTED },
        [NL80211_ATTR_SCAN_SSIDS] = { .type = NLA_NESTED },

        [NL80211_ATTR_SSID] = { .type = NLA_BINARY,
                                .len = IEEE80211_MAX_SSID_LEN },
        [NL80211_ATTR_AUTH_TYPE] = { .type = NLA_U32 },
        [NL80211_ATTR_REASON_CODE] = { .type = NLA_U16 },
        [NL80211_ATTR_FREQ_FIXED] = { .type = NLA_FLAG },
        [NL80211_ATTR_TIMED_OUT] = { .type = NLA_FLAG },
        [NL80211_ATTR_USE_MFP] = NLA_POLICY_RANGE(NLA_U32,
                                                  NL80211_MFP_NO,
                                                  NL80211_MFP_OPTIONAL),
        [NL80211_ATTR_STA_FLAGS2] =
                NLA_POLICY_EXACT_LEN_WARN(sizeof(struct nl80211_sta_flag_update)),
        [NL80211_ATTR_CONTROL_PORT] = { .type = NLA_FLAG },
        [NL80211_ATTR_CONTROL_PORT_ETHERTYPE] = { .type = NLA_U16 },
        [NL80211_ATTR_CONTROL_PORT_NO_ENCRYPT] = { .type = NLA_FLAG },
        [NL80211_ATTR_CONTROL_PORT_OVER_NL80211] = { .type = NLA_FLAG },
        [NL80211_ATTR_PRIVACY] = { .type = NLA_FLAG },
        [NL80211_ATTR_STATUS_CODE] = { .type = NLA_U16 },
        [NL80211_ATTR_CIPHER_SUITE_GROUP] = { .type = NLA_U32 },
        [NL80211_ATTR_WPA_VERSIONS] =
                NLA_POLICY_RANGE(NLA_U32, 0,
                                 NL80211_WPA_VERSION_1 |
                                 NL80211_WPA_VERSION_2 |
                                 NL80211_WPA_VERSION_3),
        [NL80211_ATTR_PID] = { .type = NLA_U32 },
        [NL80211_ATTR_4ADDR] = { .type = NLA_U8 },
        [NL80211_ATTR_PMKID] = NLA_POLICY_EXACT_LEN_WARN(WLAN_PMKID_LEN),
        [NL80211_ATTR_DURATION] = { .type = NLA_U32 },
        [NL80211_ATTR_COOKIE] = { .type = NLA_U64 },
        [NL80211_ATTR_TX_RATES] = { .type = NLA_NESTED },
        [NL80211_ATTR_FRAME] = { .type = NLA_BINARY,
                                 .len = IEEE80211_MAX_DATA_LEN },
        [NL80211_ATTR_FRAME_MATCH] = { .type = NLA_BINARY, },
        [NL80211_ATTR_PS_STATE] = NLA_POLICY_RANGE(NLA_U32,
                                                   NL80211_PS_DISABLED,
                                                   NL80211_PS_ENABLED),
        [NL80211_ATTR_CQM] = { .type = NLA_NESTED, },
        [NL80211_ATTR_LOCAL_STATE_CHANGE] = { .type = NLA_FLAG },
        [NL80211_ATTR_AP_ISOLATE] = { .type = NLA_U8 },
        [NL80211_ATTR_WIPHY_TX_POWER_SETTING] = { .type = NLA_U32 },
        [NL80211_ATTR_WIPHY_TX_POWER_LEVEL] = { .type = NLA_U32 },
        [NL80211_ATTR_FRAME_TYPE] = { .type = NLA_U16 },
        [NL80211_ATTR_WIPHY_ANTENNA_TX] = { .type = NLA_U32 },
        [NL80211_ATTR_WIPHY_ANTENNA_RX] = { .type = NLA_U32 },
        [NL80211_ATTR_MCAST_RATE] = { .type = NLA_U32 },
        [NL80211_ATTR_OFFCHANNEL_TX_OK] = { .type = NLA_FLAG },
        [NL80211_ATTR_KEY_DEFAULT_TYPES] = { .type = NLA_NESTED },
        [NL80211_ATTR_WOWLAN_TRIGGERS] = { .type = NLA_NESTED },
        [NL80211_ATTR_STA_PLINK_STATE] =
                NLA_POLICY_MAX(NLA_U8, NUM_NL80211_PLINK_STATES - 1),
        [NL80211_ATTR_MEASUREMENT_DURATION] = { .type = NLA_U16 },
        [NL80211_ATTR_MEASUREMENT_DURATION_MANDATORY] = { .type = NLA_FLAG },
        [NL80211_ATTR_MESH_PEER_AID] =
                NLA_POLICY_RANGE(NLA_U16, 1, IEEE80211_MAX_AID),
        [NL80211_ATTR_SCHED_SCAN_INTERVAL] = { .type = NLA_U32 },
        [NL80211_ATTR_REKEY_DATA] = { .type = NLA_NESTED },
        [NL80211_ATTR_SCAN_SUPP_RATES] = { .type = NLA_NESTED },
        [NL80211_ATTR_HIDDEN_SSID] =
                NLA_POLICY_RANGE(NLA_U32,
                                 NL80211_HIDDEN_SSID_NOT_IN_USE,
                                 NL80211_HIDDEN_SSID_ZERO_CONTENTS),
        [NL80211_ATTR_IE_PROBE_RESP] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_ie_attr,
                                       IEEE80211_MAX_DATA_LEN),
        [NL80211_ATTR_IE_ASSOC_RESP] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_ie_attr,
                                       IEEE80211_MAX_DATA_LEN),
        [NL80211_ATTR_ROAM_SUPPORT] = { .type = NLA_FLAG },
        [NL80211_ATTR_STA_WME] = NLA_POLICY_NESTED(nl80211_sta_wme_policy),
        [NL80211_ATTR_SCHED_SCAN_MATCH] = { .type = NLA_NESTED },
        [NL80211_ATTR_TX_NO_CCK_RATE] = { .type = NLA_FLAG },
        [NL80211_ATTR_TDLS_ACTION] = { .type = NLA_U8 },
        [NL80211_ATTR_TDLS_DIALOG_TOKEN] = { .type = NLA_U8 },
        [NL80211_ATTR_TDLS_OPERATION] = { .type = NLA_U8 },
        [NL80211_ATTR_TDLS_SUPPORT] = { .type = NLA_FLAG },
        [NL80211_ATTR_TDLS_EXTERNAL_SETUP] = { .type = NLA_FLAG },
        [NL80211_ATTR_TDLS_INITIATOR] = { .type = NLA_FLAG },
        [NL80211_ATTR_DONT_WAIT_FOR_ACK] = { .type = NLA_FLAG },
        [NL80211_ATTR_PROBE_RESP] = { .type = NLA_BINARY,
                                      .len = IEEE80211_MAX_DATA_LEN },
        [NL80211_ATTR_DFS_REGION] = { .type = NLA_U8 },
        [NL80211_ATTR_DISABLE_HT] = { .type = NLA_FLAG },
        [NL80211_ATTR_HT_CAPABILITY_MASK] = {
                .len = NL80211_HT_CAPABILITY_LEN
        },
        [NL80211_ATTR_NOACK_MAP] = { .type = NLA_U16 },
        [NL80211_ATTR_INACTIVITY_TIMEOUT] = { .type = NLA_U16 },
        [NL80211_ATTR_BG_SCAN_PERIOD] = { .type = NLA_U16 },
        [NL80211_ATTR_WDEV] = { .type = NLA_U64 },
        [NL80211_ATTR_USER_REG_HINT_TYPE] = { .type = NLA_U32 },

        /* need to include at least Auth Transaction and Status Code */
        [NL80211_ATTR_AUTH_DATA] = NLA_POLICY_MIN_LEN(4),

        [NL80211_ATTR_VHT_CAPABILITY] = NLA_POLICY_EXACT_LEN_WARN(NL80211_VHT_CAPABILITY_LEN),
        [NL80211_ATTR_SCAN_FLAGS] = { .type = NLA_U32 },
        [NL80211_ATTR_P2P_CTWINDOW] = NLA_POLICY_MAX(NLA_U8, 127),
        [NL80211_ATTR_P2P_OPPPS] = NLA_POLICY_MAX(NLA_U8, 1),
        [NL80211_ATTR_LOCAL_MESH_POWER_MODE] =
                NLA_POLICY_RANGE(NLA_U32,
                                 NL80211_MESH_POWER_UNKNOWN + 1,
                                 NL80211_MESH_POWER_MAX),
        [NL80211_ATTR_ACL_POLICY] = {. type = NLA_U32 },
        [NL80211_ATTR_MAC_ADDRS] = { .type = NLA_NESTED },
        [NL80211_ATTR_STA_CAPABILITY] = { .type = NLA_U16 },
        [NL80211_ATTR_STA_EXT_CAPABILITY] = { .type = NLA_BINARY, },
        [NL80211_ATTR_SPLIT_WIPHY_DUMP] = { .type = NLA_FLAG, },
        [NL80211_ATTR_DISABLE_VHT] = { .type = NLA_FLAG },
        [NL80211_ATTR_VHT_CAPABILITY_MASK] = {
                .len = NL80211_VHT_CAPABILITY_LEN,
        },
        [NL80211_ATTR_MDID] = { .type = NLA_U16 },
        [NL80211_ATTR_IE_RIC] = { .type = NLA_BINARY,
                                  .len = IEEE80211_MAX_DATA_LEN },
        [NL80211_ATTR_CRIT_PROT_ID] = { .type = NLA_U16 },
        [NL80211_ATTR_MAX_CRIT_PROT_DURATION] =
                NLA_POLICY_MAX(NLA_U16, NL80211_CRIT_PROTO_MAX_DURATION),
        [NL80211_ATTR_PEER_AID] =
                NLA_POLICY_RANGE(NLA_U16, 1, IEEE80211_MAX_AID),
        [NL80211_ATTR_CH_SWITCH_COUNT] = { .type = NLA_U32 },
        [NL80211_ATTR_CH_SWITCH_BLOCK_TX] = { .type = NLA_FLAG },
        [NL80211_ATTR_CSA_IES] = { .type = NLA_NESTED },
        [NL80211_ATTR_CNTDWN_OFFS_BEACON] = { .type = NLA_BINARY },
        [NL80211_ATTR_CNTDWN_OFFS_PRESP] = { .type = NLA_BINARY },
        [NL80211_ATTR_STA_SUPPORTED_CHANNELS] = NLA_POLICY_MIN_LEN(2),
        /*
         * The value of the Length field of the Supported Operating
         * Classes element is between 2 and 253.
         */
        [NL80211_ATTR_STA_SUPPORTED_OPER_CLASSES] =
                NLA_POLICY_RANGE(NLA_BINARY, 2, 253),
        [NL80211_ATTR_HANDLE_DFS] = { .type = NLA_FLAG },
        [NL80211_ATTR_OPMODE_NOTIF] = { .type = NLA_U8 },
        [NL80211_ATTR_VENDOR_ID] = { .type = NLA_U32 },
        [NL80211_ATTR_VENDOR_SUBCMD] = { .type = NLA_U32 },
        [NL80211_ATTR_VENDOR_DATA] = { .type = NLA_BINARY },
        [NL80211_ATTR_QOS_MAP] = NLA_POLICY_RANGE(NLA_BINARY,
                                                  IEEE80211_QOS_MAP_LEN_MIN,
                                                  IEEE80211_QOS_MAP_LEN_MAX),
        [NL80211_ATTR_MAC_HINT] = NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN),
        [NL80211_ATTR_WIPHY_FREQ_HINT] = { .type = NLA_U32 },
        [NL80211_ATTR_TDLS_PEER_CAPABILITY] = { .type = NLA_U32 },
        [NL80211_ATTR_SOCKET_OWNER] = { .type = NLA_FLAG },
        [NL80211_ATTR_CSA_C_OFFSETS_TX] = { .type = NLA_BINARY },
        [NL80211_ATTR_USE_RRM] = { .type = NLA_FLAG },
        [NL80211_ATTR_TSID] = NLA_POLICY_MAX(NLA_U8, IEEE80211_NUM_TIDS - 1),
        [NL80211_ATTR_USER_PRIO] =
                NLA_POLICY_MAX(NLA_U8, IEEE80211_NUM_UPS - 1),
        [NL80211_ATTR_ADMITTED_TIME] = { .type = NLA_U16 },
        [NL80211_ATTR_SMPS_MODE] = { .type = NLA_U8 },
        [NL80211_ATTR_OPER_CLASS] = { .type = NLA_U8 },
        [NL80211_ATTR_MAC_MASK] = NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN),
        [NL80211_ATTR_WIPHY_SELF_MANAGED_REG] = { .type = NLA_FLAG },
        [NL80211_ATTR_NETNS_FD] = { .type = NLA_U32 },
        [NL80211_ATTR_SCHED_SCAN_DELAY] = { .type = NLA_U32 },
        [NL80211_ATTR_REG_INDOOR] = { .type = NLA_FLAG },
        [NL80211_ATTR_PBSS] = { .type = NLA_FLAG },
        [NL80211_ATTR_BSS_SELECT] = { .type = NLA_NESTED },
        [NL80211_ATTR_STA_SUPPORT_P2P_PS] =
                NLA_POLICY_MAX(NLA_U8, NUM_NL80211_P2P_PS_STATUS - 1),
        [NL80211_ATTR_MU_MIMO_GROUP_DATA] = {
                .len = VHT_MUMIMO_GROUPS_DATA_LEN
        },
        [NL80211_ATTR_MU_MIMO_FOLLOW_MAC_ADDR] = NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN),
        [NL80211_ATTR_NAN_MASTER_PREF] = NLA_POLICY_MIN(NLA_U8, 1),
        [NL80211_ATTR_BANDS] = { .type = NLA_U32 },
        [NL80211_ATTR_NAN_CONFIG] = NLA_POLICY_NESTED(nl80211_nan_conf_policy),
        [NL80211_ATTR_NAN_FUNC] = { .type = NLA_NESTED },
        [NL80211_ATTR_FILS_KEK] = { .type = NLA_BINARY,
                                    .len = FILS_MAX_KEK_LEN },
        [NL80211_ATTR_FILS_NONCES] = NLA_POLICY_EXACT_LEN_WARN(2 * FILS_NONCE_LEN),
        [NL80211_ATTR_MULTICAST_TO_UNICAST_ENABLED] = { .type = NLA_FLAG, },
        [NL80211_ATTR_BSSID] = NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN),
        [NL80211_ATTR_SCHED_SCAN_RELATIVE_RSSI] = { .type = NLA_S8 },
        [NL80211_ATTR_SCHED_SCAN_RSSI_ADJUST] = {
                .len = sizeof(struct nl80211_bss_select_rssi_adjust)
        },
        [NL80211_ATTR_TIMEOUT_REASON] = { .type = NLA_U32 },
        [NL80211_ATTR_FILS_ERP_USERNAME] = { .type = NLA_BINARY,
                                             .len = FILS_ERP_MAX_USERNAME_LEN },
        [NL80211_ATTR_FILS_ERP_REALM] = { .type = NLA_BINARY,
                                          .len = FILS_ERP_MAX_REALM_LEN },
        [NL80211_ATTR_FILS_ERP_NEXT_SEQ_NUM] = { .type = NLA_U16 },
        [NL80211_ATTR_FILS_ERP_RRK] = { .type = NLA_BINARY,
                                        .len = FILS_ERP_MAX_RRK_LEN },
        [NL80211_ATTR_FILS_CACHE_ID] = NLA_POLICY_EXACT_LEN_WARN(2),
        [NL80211_ATTR_PMK] = { .type = NLA_BINARY, .len = PMK_MAX_LEN },
        [NL80211_ATTR_PMKR0_NAME] = NLA_POLICY_EXACT_LEN(WLAN_PMK_NAME_LEN),
        [NL80211_ATTR_SCHED_SCAN_MULTI] = { .type = NLA_FLAG },
        [NL80211_ATTR_EXTERNAL_AUTH_SUPPORT] = { .type = NLA_FLAG },

        [NL80211_ATTR_TXQ_LIMIT] = { .type = NLA_U32 },
        [NL80211_ATTR_TXQ_MEMORY_LIMIT] = { .type = NLA_U32 },
        [NL80211_ATTR_TXQ_QUANTUM] = NLA_POLICY_FULL_RANGE(NLA_U32, &q_range),
        [NL80211_ATTR_HE_CAPABILITY] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_he_capa,
                                       NL80211_HE_MAX_CAPABILITY_LEN),
        [NL80211_ATTR_FTM_RESPONDER] =
                NLA_POLICY_NESTED(nl80211_ftm_responder_policy),
        [NL80211_ATTR_TIMEOUT] = NLA_POLICY_MIN(NLA_U32, 1),
        [NL80211_ATTR_PEER_MEASUREMENTS] =
                NLA_POLICY_NESTED(nl80211_pmsr_attr_policy),
        [NL80211_ATTR_AIRTIME_WEIGHT] = NLA_POLICY_MIN(NLA_U16, 1),
        [NL80211_ATTR_SAE_PASSWORD] = { .type = NLA_BINARY,
                                        .len = SAE_PASSWORD_MAX_LEN },
        [NL80211_ATTR_TWT_RESPONDER] = { .type = NLA_FLAG },
        [NL80211_ATTR_HE_OBSS_PD] = NLA_POLICY_NESTED(he_obss_pd_policy),
        [NL80211_ATTR_VLAN_ID] = NLA_POLICY_RANGE(NLA_U16, 1, VLAN_N_VID - 2),
        [NL80211_ATTR_HE_BSS_COLOR] = NLA_POLICY_NESTED(he_bss_color_policy),
        [NL80211_ATTR_TID_CONFIG] =
                NLA_POLICY_NESTED_ARRAY(nl80211_tid_config_attr_policy),
        [NL80211_ATTR_CONTROL_PORT_NO_PREAUTH] = { .type = NLA_FLAG },
        [NL80211_ATTR_PMK_LIFETIME] = NLA_POLICY_MIN(NLA_U32, 1),
        [NL80211_ATTR_PMK_REAUTH_THRESHOLD] = NLA_POLICY_RANGE(NLA_U8, 1, 100),
        [NL80211_ATTR_RECEIVE_MULTICAST] = { .type = NLA_FLAG },
        [NL80211_ATTR_WIPHY_FREQ_OFFSET] = NLA_POLICY_RANGE(NLA_U32, 0, 999),
        [NL80211_ATTR_SCAN_FREQ_KHZ] = { .type = NLA_NESTED },
        [NL80211_ATTR_HE_6GHZ_CAPABILITY] =
                NLA_POLICY_EXACT_LEN(sizeof(struct ieee80211_he_6ghz_capa)),
        [NL80211_ATTR_FILS_DISCOVERY] =
                NLA_POLICY_NESTED(nl80211_fils_discovery_policy),
        [NL80211_ATTR_UNSOL_BCAST_PROBE_RESP] =
                NLA_POLICY_NESTED(nl80211_unsol_bcast_probe_resp_policy),
        [NL80211_ATTR_S1G_CAPABILITY] =
                NLA_POLICY_EXACT_LEN(IEEE80211_S1G_CAPABILITY_LEN),
        [NL80211_ATTR_S1G_CAPABILITY_MASK] =
                NLA_POLICY_EXACT_LEN(IEEE80211_S1G_CAPABILITY_LEN),
        [NL80211_ATTR_SAE_PWE] =
                NLA_POLICY_RANGE(NLA_U8, NL80211_SAE_PWE_HUNT_AND_PECK,
                                 NL80211_SAE_PWE_BOTH),
        [NL80211_ATTR_RECONNECT_REQUESTED] = { .type = NLA_REJECT },
        [NL80211_ATTR_SAR_SPEC] = NLA_POLICY_NESTED(sar_policy),
        [NL80211_ATTR_DISABLE_HE] = { .type = NLA_FLAG },
        [NL80211_ATTR_OBSS_COLOR_BITMAP] = { .type = NLA_U64 },
        [NL80211_ATTR_COLOR_CHANGE_COUNT] = { .type = NLA_U8 },
        [NL80211_ATTR_COLOR_CHANGE_COLOR] = { .type = NLA_U8 },
        [NL80211_ATTR_COLOR_CHANGE_ELEMS] = NLA_POLICY_NESTED(nl80211_policy),
        [NL80211_ATTR_MBSSID_CONFIG] =
                        NLA_POLICY_NESTED(nl80211_mbssid_config_policy),
        [NL80211_ATTR_MBSSID_ELEMS] = { .type = NLA_NESTED },
        [NL80211_ATTR_RADAR_BACKGROUND] = { .type = NLA_FLAG },
        [NL80211_ATTR_AP_SETTINGS_FLAGS] = { .type = NLA_U32 },
        [NL80211_ATTR_EHT_CAPABILITY] =
                NLA_POLICY_RANGE(NLA_BINARY,
                                 NL80211_EHT_MIN_CAPABILITY_LEN,
                                 NL80211_EHT_MAX_CAPABILITY_LEN),
        [NL80211_ATTR_DISABLE_EHT] = { .type = NLA_FLAG },
        [NL80211_ATTR_MLO_LINKS] =
                NLA_POLICY_NESTED_ARRAY(nl80211_policy),
        [NL80211_ATTR_MLO_LINK_ID] =
                NLA_POLICY_RANGE(NLA_U8, 0, IEEE80211_MLD_MAX_NUM_LINKS - 1),
        [NL80211_ATTR_MLD_ADDR] = NLA_POLICY_EXACT_LEN(ETH_ALEN),
        [NL80211_ATTR_MLO_SUPPORT] = { .type = NLA_FLAG },
        [NL80211_ATTR_MAX_NUM_AKM_SUITES] = { .type = NLA_REJECT },
        [NL80211_ATTR_EML_CAPABILITY] = { .type = NLA_U16 },
        [NL80211_ATTR_PUNCT_BITMAP] =
                NLA_POLICY_FULL_RANGE(NLA_U32, &nl80211_punct_bitmap_range),

        [NL80211_ATTR_MAX_HW_TIMESTAMP_PEERS] = { .type = NLA_U16 },
        [NL80211_ATTR_HW_TIMESTAMP_ENABLED] = { .type = NLA_FLAG },
        [NL80211_ATTR_EMA_RNR_ELEMS] = { .type = NLA_NESTED },
        [NL80211_ATTR_MLO_LINK_DISABLED] = { .type = NLA_FLAG },
        [NL80211_ATTR_BSS_DUMP_INCLUDE_USE_DATA] = { .type = NLA_FLAG },
        [NL80211_ATTR_MLO_TTLM_DLINK] = NLA_POLICY_EXACT_LEN(sizeof(u16) * 8),
        [NL80211_ATTR_MLO_TTLM_ULINK] = NLA_POLICY_EXACT_LEN(sizeof(u16) * 8),
        [NL80211_ATTR_ASSOC_SPP_AMSDU] = { .type = NLA_FLAG },
        [NL80211_ATTR_VIF_RADIO_MASK] = { .type = NLA_U32 },
        [NL80211_ATTR_SUPPORTED_SELECTORS] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_supported_selectors,
                                       NL80211_MAX_SUPP_SELECTORS),
        [NL80211_ATTR_MLO_RECONF_REM_LINKS] = { .type = NLA_U16 },
        [NL80211_ATTR_EPCS] = { .type = NLA_FLAG },
        [NL80211_ATTR_EXT_MLD_CAPA_AND_OPS] = { .type = NLA_U16 },
        [NL80211_ATTR_WIPHY_RADIO_INDEX] = { .type = NLA_U8 },
        [NL80211_ATTR_S1G_LONG_BEACON_PERIOD] = NLA_POLICY_MIN(NLA_U8, 2),
        [NL80211_ATTR_S1G_SHORT_BEACON] =
                NLA_POLICY_NESTED(nl80211_s1g_short_beacon),
        [NL80211_ATTR_BSS_PARAM] = { .type = NLA_FLAG },
        [NL80211_ATTR_S1G_PRIMARY_2MHZ] = { .type = NLA_FLAG },
        [NL80211_ATTR_EPP_PEER] = { .type = NLA_FLAG },
        [NL80211_ATTR_UHR_CAPABILITY] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_uhr_capa, 255),
        [NL80211_ATTR_DISABLE_UHR] = { .type = NLA_FLAG },
        [NL80211_ATTR_UHR_OPERATION] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_uhr_operation),
        [NL80211_ATTR_NAN_CHANNEL] = NLA_POLICY_NESTED(nl80211_policy),
        [NL80211_ATTR_NAN_CHANNEL_ENTRY] = NLA_POLICY_EXACT_LEN(6),
        [NL80211_ATTR_NAN_RX_NSS] = { .type = NLA_U8 },
        [NL80211_ATTR_NAN_TIME_SLOTS] =
                NLA_POLICY_EXACT_LEN(CFG80211_NAN_SCHED_NUM_TIME_SLOTS),
        [NL80211_ATTR_NAN_AVAIL_BLOB] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_nan_avail_blob),
        [NL80211_ATTR_NAN_SCHED_DEFERRED] = { .type = NLA_FLAG },
        [NL80211_ATTR_NAN_NMI_MAC] = NLA_POLICY_ETH_ADDR,
        [NL80211_ATTR_NAN_ULW] =
                NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_nan_ulw),
        [NL80211_ATTR_NAN_COMMITTED_DW] = { .type = NLA_U16 },
        [NL80211_ATTR_NAN_SEQ_ID] = { .type = NLA_U8 },
        [NL80211_ATTR_NAN_MAX_CHAN_SWITCH_TIME] = { .type = NLA_U16 },
        [NL80211_ATTR_NAN_PEER_MAPS] =
                NLA_POLICY_NESTED_ARRAY(nl80211_nan_peer_map_policy),
        [NL80211_ATTR_NPCA_PRIMARY_FREQ] = { .type = NLA_U32 },
        [NL80211_ATTR_NPCA_PUNCT_BITMAP] =
                NLA_POLICY_FULL_RANGE(NLA_U32, &nl80211_punct_bitmap_range),
        [NL80211_ATTR_STA_DUMP_LINK_STATS] = { .type = NLA_FLAG },
};

/* policy for the key attributes */
static const struct nla_policy nl80211_key_policy[NL80211_KEY_MAX + 1] = {
        [NL80211_KEY_DATA] = { .type = NLA_BINARY, .len = WLAN_MAX_KEY_LEN },
        [NL80211_KEY_IDX] = { .type = NLA_U8 },
        [NL80211_KEY_CIPHER] = { .type = NLA_U32 },
        [NL80211_KEY_SEQ] = { .type = NLA_BINARY, .len = 16 },
        [NL80211_KEY_DEFAULT] = { .type = NLA_FLAG },
        [NL80211_KEY_DEFAULT_MGMT] = { .type = NLA_FLAG },
        [NL80211_KEY_TYPE] = NLA_POLICY_MAX(NLA_U32, NUM_NL80211_KEYTYPES - 1),
        [NL80211_KEY_DEFAULT_TYPES] = { .type = NLA_NESTED },
        [NL80211_KEY_MODE] = NLA_POLICY_RANGE(NLA_U8, 0, NL80211_KEY_SET_TX),
        [NL80211_KEY_LTF_SEED] = {
                .type = NLA_BINARY,
                .len = WLAN_MAX_SECURE_LTF_KEYSEED_LEN,
        },
};

/* policy for the key default flags */
static const struct nla_policy
nl80211_key_default_policy[NUM_NL80211_KEY_DEFAULT_TYPES] = {
        [NL80211_KEY_DEFAULT_TYPE_UNICAST] = { .type = NLA_FLAG },
        [NL80211_KEY_DEFAULT_TYPE_MULTICAST] = { .type = NLA_FLAG },
};

#ifdef CONFIG_PM
/* policy for WoWLAN attributes */
static const struct nla_policy
nl80211_wowlan_policy[NUM_NL80211_WOWLAN_TRIG] = {
        [NL80211_WOWLAN_TRIG_ANY] = { .type = NLA_FLAG },
        [NL80211_WOWLAN_TRIG_DISCONNECT] = { .type = NLA_FLAG },
        [NL80211_WOWLAN_TRIG_MAGIC_PKT] = { .type = NLA_FLAG },
        [NL80211_WOWLAN_TRIG_PKT_PATTERN] = { .type = NLA_NESTED },
        [NL80211_WOWLAN_TRIG_GTK_REKEY_FAILURE] = { .type = NLA_FLAG },
        [NL80211_WOWLAN_TRIG_EAP_IDENT_REQUEST] = { .type = NLA_FLAG },
        [NL80211_WOWLAN_TRIG_4WAY_HANDSHAKE] = { .type = NLA_FLAG },
        [NL80211_WOWLAN_TRIG_RFKILL_RELEASE] = { .type = NLA_FLAG },
        [NL80211_WOWLAN_TRIG_TCP_CONNECTION] = { .type = NLA_NESTED },
        [NL80211_WOWLAN_TRIG_NET_DETECT] = { .type = NLA_NESTED },
};

static const struct nla_policy
nl80211_wowlan_tcp_policy[NUM_NL80211_WOWLAN_TCP] = {
        [NL80211_WOWLAN_TCP_SRC_IPV4] = { .type = NLA_U32 },
        [NL80211_WOWLAN_TCP_DST_IPV4] = { .type = NLA_U32 },
        [NL80211_WOWLAN_TCP_DST_MAC] = NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN),
        [NL80211_WOWLAN_TCP_SRC_PORT] = { .type = NLA_U16 },
        [NL80211_WOWLAN_TCP_DST_PORT] = { .type = NLA_U16 },
        [NL80211_WOWLAN_TCP_DATA_PAYLOAD] = NLA_POLICY_MIN_LEN(1),
        [NL80211_WOWLAN_TCP_DATA_PAYLOAD_SEQ] = {
                .len = sizeof(struct nl80211_wowlan_tcp_data_seq)
        },
        [NL80211_WOWLAN_TCP_DATA_PAYLOAD_TOKEN] = {
                .len = sizeof(struct nl80211_wowlan_tcp_data_token)
        },
        [NL80211_WOWLAN_TCP_DATA_INTERVAL] = { .type = NLA_U32 },
        [NL80211_WOWLAN_TCP_WAKE_PAYLOAD] = NLA_POLICY_MIN_LEN(1),
        [NL80211_WOWLAN_TCP_WAKE_MASK] = NLA_POLICY_MIN_LEN(1),
};
#endif /* CONFIG_PM */

/* policy for coalesce rule attributes */
static const struct nla_policy
nl80211_coalesce_policy[NUM_NL80211_ATTR_COALESCE_RULE] = {
        [NL80211_ATTR_COALESCE_RULE_DELAY] = { .type = NLA_U32 },
        [NL80211_ATTR_COALESCE_RULE_CONDITION] =
                NLA_POLICY_RANGE(NLA_U32,
                                 NL80211_COALESCE_CONDITION_MATCH,
                                 NL80211_COALESCE_CONDITION_NO_MATCH),
        [NL80211_ATTR_COALESCE_RULE_PKT_PATTERN] = { .type = NLA_NESTED },
};

/* policy for GTK rekey offload attributes */
static const struct nla_policy
nl80211_rekey_policy[NUM_NL80211_REKEY_DATA] = {
        [NL80211_REKEY_DATA_KEK] = {
                .type = NLA_BINARY,
                .len = NL80211_KEK_EXT_LEN
        },
        [NL80211_REKEY_DATA_KCK] = {
                .type = NLA_BINARY,
                .len = NL80211_KCK_EXT_LEN_32
        },
        [NL80211_REKEY_DATA_REPLAY_CTR] = NLA_POLICY_EXACT_LEN(NL80211_REPLAY_CTR_LEN),
        [NL80211_REKEY_DATA_AKM] = { .type = NLA_U32 },
};

static const struct nla_policy
nl80211_match_policy[NL80211_SCHED_SCAN_MATCH_ATTR_MAX + 1] = {
        [NL80211_SCHED_SCAN_MATCH_ATTR_SSID] = { .type = NLA_BINARY,
                                                 .len = IEEE80211_MAX_SSID_LEN },
        [NL80211_SCHED_SCAN_MATCH_ATTR_BSSID] = NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN),
        [NL80211_SCHED_SCAN_MATCH_ATTR_RSSI] = { .type = NLA_U32 },
};

static const struct nla_policy
nl80211_plan_policy[NL80211_SCHED_SCAN_PLAN_MAX + 1] = {
        [NL80211_SCHED_SCAN_PLAN_INTERVAL] = { .type = NLA_U32 },
        [NL80211_SCHED_SCAN_PLAN_ITERATIONS] = { .type = NLA_U32 },
};

static const struct nla_policy
nl80211_bss_select_policy[NL80211_BSS_SELECT_ATTR_MAX + 1] = {
        [NL80211_BSS_SELECT_ATTR_RSSI] = { .type = NLA_FLAG },
        [NL80211_BSS_SELECT_ATTR_BAND_PREF] = { .type = NLA_U32 },
        [NL80211_BSS_SELECT_ATTR_RSSI_ADJUST] = {
                .len = sizeof(struct nl80211_bss_select_rssi_adjust)
        },
};

/* policy for NAN function attributes */
static const struct nla_policy
nl80211_nan_func_policy[NL80211_NAN_FUNC_ATTR_MAX + 1] = {
        [NL80211_NAN_FUNC_TYPE] =
                NLA_POLICY_MAX(NLA_U8, NL80211_NAN_FUNC_MAX_TYPE),
        [NL80211_NAN_FUNC_SERVICE_ID] = {
                                    .len = NL80211_NAN_FUNC_SERVICE_ID_LEN },
        [NL80211_NAN_FUNC_PUBLISH_TYPE] = { .type = NLA_U8 },
        [NL80211_NAN_FUNC_PUBLISH_BCAST] = { .type = NLA_FLAG },
        [NL80211_NAN_FUNC_SUBSCRIBE_ACTIVE] = { .type = NLA_FLAG },
        [NL80211_NAN_FUNC_FOLLOW_UP_ID] = { .type = NLA_U8 },
        [NL80211_NAN_FUNC_FOLLOW_UP_REQ_ID] = { .type = NLA_U8 },
        [NL80211_NAN_FUNC_FOLLOW_UP_DEST] = NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN),
        [NL80211_NAN_FUNC_CLOSE_RANGE] = { .type = NLA_FLAG },
        [NL80211_NAN_FUNC_TTL] = { .type = NLA_U32 },
        [NL80211_NAN_FUNC_SERVICE_INFO] = { .type = NLA_BINARY,
                        .len = NL80211_NAN_FUNC_SERVICE_SPEC_INFO_MAX_LEN },
        [NL80211_NAN_FUNC_SRF] = { .type = NLA_NESTED },
        [NL80211_NAN_FUNC_RX_MATCH_FILTER] = { .type = NLA_NESTED },
        [NL80211_NAN_FUNC_TX_MATCH_FILTER] = { .type = NLA_NESTED },
        [NL80211_NAN_FUNC_INSTANCE_ID] = { .type = NLA_U8 },
        [NL80211_NAN_FUNC_TERM_REASON] = { .type = NLA_U8 },
};

/* policy for Service Response Filter attributes */
static const struct nla_policy
nl80211_nan_srf_policy[NL80211_NAN_SRF_ATTR_MAX + 1] = {
        [NL80211_NAN_SRF_INCLUDE] = { .type = NLA_FLAG },
        [NL80211_NAN_SRF_BF] = { .type = NLA_BINARY,
                                 .len =  NL80211_NAN_FUNC_SRF_MAX_LEN },
        [NL80211_NAN_SRF_BF_IDX] = { .type = NLA_U8 },
        [NL80211_NAN_SRF_MAC_ADDRS] = { .type = NLA_NESTED },
};

/* policy for packet pattern attributes */
static const struct nla_policy
nl80211_packet_pattern_policy[MAX_NL80211_PKTPAT + 1] = {
        [NL80211_PKTPAT_MASK] = { .type = NLA_BINARY, },
        [NL80211_PKTPAT_PATTERN] = { .type = NLA_BINARY, },
        [NL80211_PKTPAT_OFFSET] = { .type = NLA_U32 },
};

static int nl80211_prepare_wdev_dump(struct netlink_callback *cb,
                                     struct cfg80211_registered_device **rdev,
                                     struct wireless_dev **wdev,
                                     struct nlattr **attrbuf)
{
        int err;

        if (!cb->args[0]) {
                struct nlattr **attrbuf_free = NULL;

                if (!attrbuf) {
                        attrbuf = kzalloc_objs(*attrbuf, NUM_NL80211_ATTR);
                        if (!attrbuf)
                                return -ENOMEM;
                        attrbuf_free = attrbuf;
                }

                err = nlmsg_parse_deprecated(cb->nlh,
                                             GENL_HDRLEN + nl80211_fam.hdrsize,
                                             attrbuf, nl80211_fam.maxattr,
                                             nl80211_policy, NULL);
                if (err) {
                        kfree(attrbuf_free);
                        return err;
                }

                rtnl_lock();
                *wdev = __cfg80211_wdev_from_attrs(NULL, sock_net(cb->skb->sk),
                                                   attrbuf);
                kfree(attrbuf_free);
                if (IS_ERR(*wdev)) {
                        rtnl_unlock();
                        return PTR_ERR(*wdev);
                }
                *rdev = wiphy_to_rdev((*wdev)->wiphy);
                mutex_lock(&(*rdev)->wiphy.mtx);
                rtnl_unlock();
                /* 0 is the first index - add 1 to parse only once */
                cb->args[0] = (*rdev)->wiphy_idx + 1;
                cb->args[1] = (*wdev)->identifier;
        } else {
                /* subtract the 1 again here */
                struct wiphy *wiphy;
                struct wireless_dev *tmp;

                rtnl_lock();
                wiphy = wiphy_idx_to_wiphy(cb->args[0] - 1);
                if (!wiphy) {
                        rtnl_unlock();
                        return -ENODEV;
                }

                /*
                 * The first invocation validated the wdev's netns against
                 * the caller via __cfg80211_wdev_from_attrs(). The wiphy
                 * may have moved netns between dumpit invocations (via
                 * NL80211_CMD_SET_WIPHY_NETNS), so re-check here.
                 */
                if (!net_eq(wiphy_net(wiphy), sock_net(cb->skb->sk))) {
                        rtnl_unlock();
                        return -ENODEV;
                }

                *rdev = wiphy_to_rdev(wiphy);
                *wdev = NULL;

                list_for_each_entry(tmp, &(*rdev)->wiphy.wdev_list, list) {
                        if (tmp->identifier == cb->args[1]) {
                                *wdev = tmp;
                                break;
                        }
                }

                if (!*wdev) {
                        rtnl_unlock();
                        return -ENODEV;
                }
                mutex_lock(&(*rdev)->wiphy.mtx);
                rtnl_unlock();
        }

        return 0;
}

/* message building helper */
void *nl80211hdr_put(struct sk_buff *skb, u32 portid, u32 seq,
                     int flags, u8 cmd)
{
        /* since there is no private header just add the generic one */
        return genlmsg_put(skb, portid, seq, &nl80211_fam, flags, cmd);
}

static int nl80211_msg_put_wmm_rules(struct sk_buff *msg,
                                     const struct ieee80211_reg_rule *rule)
{
        int j;
        struct nlattr *nl_wmm_rules =
                nla_nest_start_noflag(msg, NL80211_FREQUENCY_ATTR_WMM);

        if (!nl_wmm_rules)
                goto nla_put_failure;

        for (j = 0; j < IEEE80211_NUM_ACS; j++) {
                struct nlattr *nl_wmm_rule = nla_nest_start_noflag(msg, j);

                if (!nl_wmm_rule)
                        goto nla_put_failure;

                if (nla_put_u16(msg, NL80211_WMMR_CW_MIN,
                                rule->wmm_rule.client[j].cw_min) ||
                    nla_put_u16(msg, NL80211_WMMR_CW_MAX,
                                rule->wmm_rule.client[j].cw_max) ||
                    nla_put_u8(msg, NL80211_WMMR_AIFSN,
                               rule->wmm_rule.client[j].aifsn) ||
                    nla_put_u16(msg, NL80211_WMMR_TXOP,
                                rule->wmm_rule.client[j].cot))
                        goto nla_put_failure;

                nla_nest_end(msg, nl_wmm_rule);
        }
        nla_nest_end(msg, nl_wmm_rules);

        return 0;

nla_put_failure:
        return -ENOBUFS;
}

static int nl80211_msg_put_channel(struct sk_buff *msg, struct wiphy *wiphy,
                                   struct ieee80211_channel *chan,
                                   bool large)
{
        /* Some channels must be completely excluded from the
         * list to protect old user-space tools from breaking
         */
        if (!large && chan->flags &
            (IEEE80211_CHAN_NO_10MHZ | IEEE80211_CHAN_NO_20MHZ))
                return 0;
        if (!large && chan->freq_offset)
                return 0;

        if (nla_put_u32(msg, NL80211_FREQUENCY_ATTR_FREQ,
                        chan->center_freq))
                goto nla_put_failure;

        if (nla_put_u32(msg, NL80211_FREQUENCY_ATTR_OFFSET, chan->freq_offset))
                goto nla_put_failure;

        if ((chan->flags & IEEE80211_CHAN_PSD) &&
            nla_put_s8(msg, NL80211_FREQUENCY_ATTR_PSD, chan->psd))
                goto nla_put_failure;

        if ((chan->flags & IEEE80211_CHAN_DISABLED) &&
            nla_put_flag(msg, NL80211_FREQUENCY_ATTR_DISABLED))
                goto nla_put_failure;
        if (chan->flags & IEEE80211_CHAN_NO_IR) {
                if (nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_IR))
                        goto nla_put_failure;
                if (nla_put_flag(msg, __NL80211_FREQUENCY_ATTR_NO_IBSS))
                        goto nla_put_failure;
        }
        if (chan->flags & IEEE80211_CHAN_RADAR) {
                if (nla_put_flag(msg, NL80211_FREQUENCY_ATTR_RADAR))
                        goto nla_put_failure;
                if (large) {
                        u32 time;

                        time = elapsed_jiffies_msecs(chan->dfs_state_entered);

                        if (nla_put_u32(msg, NL80211_FREQUENCY_ATTR_DFS_STATE,
                                        chan->dfs_state))
                                goto nla_put_failure;
                        if (nla_put_u32(msg, NL80211_FREQUENCY_ATTR_DFS_TIME,
                                        time))
                                goto nla_put_failure;
                        if (nla_put_u32(msg,
                                        NL80211_FREQUENCY_ATTR_DFS_CAC_TIME,
                                        chan->dfs_cac_ms))
                                goto nla_put_failure;
                }
        }

        if (large) {
                if ((chan->flags & IEEE80211_CHAN_NO_HT40MINUS) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_HT40_MINUS))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_HT40PLUS) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_HT40_PLUS))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_80MHZ) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_80MHZ))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_160MHZ) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_160MHZ))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_INDOOR_ONLY) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_INDOOR_ONLY))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_IR_CONCURRENT) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_IR_CONCURRENT))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_20MHZ) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_20MHZ))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_10MHZ) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_10MHZ))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_HE) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_HE))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_320MHZ) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_320MHZ))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_EHT) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_EHT))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_DFS_CONCURRENT) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_DFS_CONCURRENT))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_6GHZ_VLP_CLIENT))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_6GHZ_AFC_CLIENT) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_6GHZ_AFC_CLIENT))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_CAN_MONITOR) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_CAN_MONITOR))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_ALLOW_6GHZ_VLP_AP) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_ALLOW_6GHZ_VLP_AP))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_ALLOW_20MHZ_ACTIVITY) &&
                    nla_put_flag(msg,
                                 NL80211_FREQUENCY_ATTR_ALLOW_20MHZ_ACTIVITY))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_4MHZ) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_4MHZ))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_8MHZ) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_8MHZ))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_16MHZ) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_16MHZ))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_S1G_NO_PRIMARY) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_S1G_NO_PRIMARY))
                        goto nla_put_failure;
                if ((chan->flags & IEEE80211_CHAN_NO_UHR) &&
                    nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_UHR))
                        goto nla_put_failure;
                if (chan->cac_start_time &&
                    nla_put_u64_64bit(msg,
                                      NL80211_FREQUENCY_ATTR_CAC_START_TIME,
                                      chan->cac_start_time,
                                      NL80211_FREQUENCY_ATTR_PAD))
                        goto nla_put_failure;
        }

        if (nla_put_u32(msg, NL80211_FREQUENCY_ATTR_MAX_TX_POWER,
                        DBM_TO_MBM(chan->max_power)))
                goto nla_put_failure;

        if (large) {
                const struct ieee80211_reg_rule *rule =
                        freq_reg_info(wiphy, MHZ_TO_KHZ(chan->center_freq));

                if (!IS_ERR_OR_NULL(rule) && rule->has_wmm) {
                        if (nl80211_msg_put_wmm_rules(msg, rule))
                                goto nla_put_failure;
                }
        }

        return 0;

 nla_put_failure:
        return -ENOBUFS;
}

static bool nl80211_put_txq_stats(struct sk_buff *msg,
                                  struct cfg80211_txq_stats *txqstats,
                                  int attrtype)
{
        struct nlattr *txqattr;

#define PUT_TXQVAL_U32(attr, memb) do {                                          \
        if (txqstats->filled & BIT(NL80211_TXQ_STATS_ ## attr) &&          \
            nla_put_u32(msg, NL80211_TXQ_STATS_ ## attr, txqstats->memb)) \
                return false;                                                  \
        } while (0)

        txqattr = nla_nest_start_noflag(msg, attrtype);
        if (!txqattr)
                return false;

        PUT_TXQVAL_U32(BACKLOG_BYTES, backlog_bytes);
        PUT_TXQVAL_U32(BACKLOG_PACKETS, backlog_packets);
        PUT_TXQVAL_U32(FLOWS, flows);
        PUT_TXQVAL_U32(DROPS, drops);
        PUT_TXQVAL_U32(ECN_MARKS, ecn_marks);
        PUT_TXQVAL_U32(OVERLIMIT, overlimit);
        PUT_TXQVAL_U32(OVERMEMORY, overmemory);
        PUT_TXQVAL_U32(COLLISIONS, collisions);
        PUT_TXQVAL_U32(TX_BYTES, tx_bytes);
        PUT_TXQVAL_U32(TX_PACKETS, tx_packets);
        PUT_TXQVAL_U32(MAX_FLOWS, max_flows);
        nla_nest_end(msg, txqattr);

#undef PUT_TXQVAL_U32
        return true;
}

/* netlink command implementations */

/**
 * nl80211_link_id - return link ID
 * @attrs: attributes to look at
 *
 * Returns: the link ID or 0 if not given
 *
 * Note this function doesn't do any validation of the link
 * ID validity wrt. links that were actually added, so it must
 * be called only from ops with %NL80211_FLAG_MLO_VALID_LINK_ID
 * or if additional validation is done.
 */
static unsigned int nl80211_link_id(struct nlattr **attrs)
{
        struct nlattr *linkid = attrs[NL80211_ATTR_MLO_LINK_ID];

        return nla_get_u8_default(linkid, 0);
}

static int nl80211_link_id_or_invalid(struct nlattr **attrs)
{
        struct nlattr *linkid = attrs[NL80211_ATTR_MLO_LINK_ID];

        if (!linkid)
                return -1;

        return nla_get_u8(linkid);
}

struct key_parse {
        struct key_params p;
        int idx;
        int type;
        bool def, defmgmt, defbeacon;
        bool def_uni, def_multi;
};

static int nl80211_parse_key_new(struct genl_info *info, struct nlattr *key,
                                 struct key_parse *k)
{
        struct nlattr *tb[NL80211_KEY_MAX + 1];
        int err = nla_parse_nested_deprecated(tb, NL80211_KEY_MAX, key,
                                              nl80211_key_policy,
                                              info->extack);
        if (err)
                return err;

        k->def = !!tb[NL80211_KEY_DEFAULT];
        k->defmgmt = !!tb[NL80211_KEY_DEFAULT_MGMT];
        k->defbeacon = !!tb[NL80211_KEY_DEFAULT_BEACON];

        if (k->def) {
                k->def_uni = true;
                k->def_multi = true;
        }
        if (k->defmgmt || k->defbeacon)
                k->def_multi = true;

        if (tb[NL80211_KEY_IDX])
                k->idx = nla_get_u8(tb[NL80211_KEY_IDX]);

        if (tb[NL80211_KEY_DATA]) {
                k->p.key = nla_data(tb[NL80211_KEY_DATA]);
                k->p.key_len = nla_len(tb[NL80211_KEY_DATA]);
        }

        if (tb[NL80211_KEY_SEQ]) {
                k->p.seq = nla_data(tb[NL80211_KEY_SEQ]);
                k->p.seq_len = nla_len(tb[NL80211_KEY_SEQ]);
        }

        if (tb[NL80211_KEY_CIPHER])
                k->p.cipher = nla_get_u32(tb[NL80211_KEY_CIPHER]);

        if (tb[NL80211_KEY_TYPE])
                k->type = nla_get_u32(tb[NL80211_KEY_TYPE]);

        if (tb[NL80211_KEY_DEFAULT_TYPES]) {
                struct nlattr *kdt[NUM_NL80211_KEY_DEFAULT_TYPES];

                err = nla_parse_nested_deprecated(kdt,
                                                  NUM_NL80211_KEY_DEFAULT_TYPES - 1,
                                                  tb[NL80211_KEY_DEFAULT_TYPES],
                                                  nl80211_key_default_policy,
                                                  info->extack);
                if (err)
                        return err;

                k->def_uni = kdt[NL80211_KEY_DEFAULT_TYPE_UNICAST];
                k->def_multi = kdt[NL80211_KEY_DEFAULT_TYPE_MULTICAST];
        }

        if (tb[NL80211_KEY_MODE])
                k->p.mode = nla_get_u8(tb[NL80211_KEY_MODE]);

        if (tb[NL80211_KEY_LTF_SEED]) {
                k->p.ltf_keyseed = nla_data(tb[NL80211_KEY_LTF_SEED]);
                k->p.ltf_keyseed_len = nla_len(tb[NL80211_KEY_LTF_SEED]);
        }

        return 0;
}

static int nl80211_parse_key_old(struct genl_info *info, struct key_parse *k)
{
        if (info->attrs[NL80211_ATTR_KEY_DATA]) {
                k->p.key = nla_data(info->attrs[NL80211_ATTR_KEY_DATA]);
                k->p.key_len = nla_len(info->attrs[NL80211_ATTR_KEY_DATA]);
        }

        if (info->attrs[NL80211_ATTR_KEY_SEQ]) {
                k->p.seq = nla_data(info->attrs[NL80211_ATTR_KEY_SEQ]);
                k->p.seq_len = nla_len(info->attrs[NL80211_ATTR_KEY_SEQ]);
        }

        if (info->attrs[NL80211_ATTR_KEY_IDX])
                k->idx = nla_get_u8(info->attrs[NL80211_ATTR_KEY_IDX]);

        if (info->attrs[NL80211_ATTR_KEY_CIPHER])
                k->p.cipher = nla_get_u32(info->attrs[NL80211_ATTR_KEY_CIPHER]);

        k->def = !!info->attrs[NL80211_ATTR_KEY_DEFAULT];
        k->defmgmt = !!info->attrs[NL80211_ATTR_KEY_DEFAULT_MGMT];

        if (k->def) {
                k->def_uni = true;
                k->def_multi = true;
        }
        if (k->defmgmt)
                k->def_multi = true;

        if (info->attrs[NL80211_ATTR_KEY_TYPE])
                k->type = nla_get_u32(info->attrs[NL80211_ATTR_KEY_TYPE]);

        if (info->attrs[NL80211_ATTR_KEY_DEFAULT_TYPES]) {
                struct nlattr *kdt[NUM_NL80211_KEY_DEFAULT_TYPES];
                int err = nla_parse_nested_deprecated(kdt,
                                                      NUM_NL80211_KEY_DEFAULT_TYPES - 1,
                                                      info->attrs[NL80211_ATTR_KEY_DEFAULT_TYPES],
                                                      nl80211_key_default_policy,
                                                      info->extack);
                if (err)
                        return err;

                k->def_uni = kdt[NL80211_KEY_DEFAULT_TYPE_UNICAST];
                k->def_multi = kdt[NL80211_KEY_DEFAULT_TYPE_MULTICAST];
        }

        return 0;
}

static int nl80211_parse_key(struct genl_info *info, struct key_parse *k)
{
        int err;

        memset(k, 0, sizeof(*k));
        k->idx = -1;
        k->type = -1;

        if (info->attrs[NL80211_ATTR_KEY])
                err = nl80211_parse_key_new(info, info->attrs[NL80211_ATTR_KEY], k);
        else
                err = nl80211_parse_key_old(info, k);

        if (err)
                return err;

        if ((k->def ? 1 : 0) + (k->defmgmt ? 1 : 0) +
            (k->defbeacon ? 1 : 0) > 1) {
                GENL_SET_ERR_MSG(info,
                                 "key with multiple default flags is invalid");
                return -EINVAL;
        }

        if (k->defmgmt || k->defbeacon) {
                if (k->def_uni || !k->def_multi) {
                        GENL_SET_ERR_MSG(info,
                                         "defmgmt/defbeacon key must be mcast");
                        return -EINVAL;
                }
        }

        if (k->idx != -1) {
                if (k->defmgmt) {
                        if (k->idx < 4 || k->idx > 5) {
                                GENL_SET_ERR_MSG(info,
                                                 "defmgmt key idx not 4 or 5");
                                return -EINVAL;
                        }
                } else if (k->defbeacon) {
                        if (k->idx < 6 || k->idx > 7) {
                                GENL_SET_ERR_MSG(info,
                                                 "defbeacon key idx not 6 or 7");
                                return -EINVAL;
                        }
                } else if (k->def) {
                        if (k->idx < 0 || k->idx > 3) {
                                GENL_SET_ERR_MSG(info, "def key idx not 0-3");
                                return -EINVAL;
                        }
                } else {
                        if (k->idx < 0 || k->idx > 7) {
                                GENL_SET_ERR_MSG(info, "key idx not 0-7");
                                return -EINVAL;
                        }
                }
        }

        return 0;
}

static struct cfg80211_cached_keys *
nl80211_parse_connkeys(struct cfg80211_registered_device *rdev,
                       struct wireless_dev *wdev,
                       struct genl_info *info, bool *no_ht)
{
        struct nlattr *keys = info->attrs[NL80211_ATTR_KEYS];
        struct key_parse parse;
        struct nlattr *key;
        struct cfg80211_cached_keys *result;
        int rem, err, def = 0;
        bool have_key = false;

        nla_for_each_nested(key, keys, rem) {
                have_key = true;
                break;
        }

        if (!have_key)
                return NULL;

        result = kzalloc_obj(*result);
        if (!result)
                return ERR_PTR(-ENOMEM);

        result->def = -1;

        nla_for_each_nested(key, keys, rem) {
                memset(&parse, 0, sizeof(parse));
                parse.idx = -1;

                err = nl80211_parse_key_new(info, key, &parse);
                if (err)
                        goto error;
                err = -EINVAL;
                if (!parse.p.key)
                        goto error;
                if (parse.idx < 0 || parse.idx > 3) {
                        GENL_SET_ERR_MSG(info, "key index out of range [0-3]");
                        goto error;
                }
                if (parse.def) {
                        if (def) {
                                GENL_SET_ERR_MSG(info,
                                                 "only one key can be default");
                                goto error;
                        }
                        def = 1;
                        result->def = parse.idx;
                        if (!parse.def_uni || !parse.def_multi)
                                goto error;
                } else if (parse.defmgmt)
                        goto error;
                err = cfg80211_validate_key_settings(rdev, wdev, &parse.p,
                                                     parse.idx, false, NULL);
                if (err)
                        goto error;
                if (parse.p.cipher != WLAN_CIPHER_SUITE_WEP40 &&
                    parse.p.cipher != WLAN_CIPHER_SUITE_WEP104) {
                        GENL_SET_ERR_MSG(info, "connect key must be WEP");
                        err = -EINVAL;
                        goto error;
                }
                result->params[parse.idx].cipher = parse.p.cipher;
                result->params[parse.idx].key_len = parse.p.key_len;
                result->params[parse.idx].key = result->data[parse.idx];
                memcpy(result->data[parse.idx], parse.p.key, parse.p.key_len);

                /* must be WEP key if we got here */
                if (no_ht)
                        *no_ht = true;
        }

        if (result->def < 0) {
                err = -EINVAL;
                GENL_SET_ERR_MSG(info, "need a default/TX key");
                goto error;
        }

        return result;
 error:
        kfree_sensitive(result);
        return ERR_PTR(err);
}

static int nl80211_key_allowed(struct wireless_dev *wdev)
{
        lockdep_assert_wiphy(wdev->wiphy);

        switch (wdev->iftype) {
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_AP_VLAN:
        case NL80211_IFTYPE_P2P_GO:
        case NL80211_IFTYPE_MESH_POINT:
                break;
        case NL80211_IFTYPE_ADHOC:
                if (wdev->u.ibss.current_bss)
                        return 0;
                return -ENOLINK;
        case NL80211_IFTYPE_STATION:
        case NL80211_IFTYPE_P2P_CLIENT:
                if (wdev->connected ||
                    (wiphy_ext_feature_isset(wdev->wiphy,
                                             NL80211_EXT_FEATURE_ASSOC_FRAME_ENCRYPTION)))
                        return 0;
                return -ENOLINK;
        case NL80211_IFTYPE_NAN:
        case NL80211_IFTYPE_NAN_DATA:
                if (wiphy_ext_feature_isset(wdev->wiphy,
                                            NL80211_EXT_FEATURE_SECURE_NAN))
                        return 0;
                return -EINVAL;
        case NL80211_IFTYPE_PD:
                if (wiphy_ext_feature_isset(wdev->wiphy,
                                            NL80211_EXT_FEATURE_SECURE_RTT))
                        return 0;
                return -EINVAL;
        case NL80211_IFTYPE_UNSPECIFIED:
        case NL80211_IFTYPE_OCB:
        case NL80211_IFTYPE_MONITOR:
        case NL80211_IFTYPE_P2P_DEVICE:
        case NL80211_IFTYPE_WDS:
        case NUM_NL80211_IFTYPES:
                return -EINVAL;
        }

        return 0;
}

static struct ieee80211_channel *nl80211_get_valid_chan(struct wiphy *wiphy,
                                                        u32 freq)
{
        struct ieee80211_channel *chan;

        chan = ieee80211_get_channel_khz(wiphy, freq);
        if (!chan || chan->flags & IEEE80211_CHAN_DISABLED)
                return NULL;
        return chan;
}

static int nl80211_put_iftypes(struct sk_buff *msg, u32 attr, u16 ifmodes)
{
        struct nlattr *nl_modes = nla_nest_start_noflag(msg, attr);
        int i;

        if (!nl_modes)
                goto nla_put_failure;

        i = 0;
        while (ifmodes) {
                if ((ifmodes & 1) && nla_put_flag(msg, i))
                        goto nla_put_failure;
                ifmodes >>= 1;
                i++;
        }

        nla_nest_end(msg, nl_modes);
        return 0;

nla_put_failure:
        return -ENOBUFS;
}

static int nl80211_put_ifcomb_data(struct sk_buff *msg, bool large, int idx,
                                   const struct ieee80211_iface_combination *c,
                                   u16 nested)
{
        struct nlattr *nl_combi, *nl_limits;
        int i;

        nl_combi = nla_nest_start_noflag(msg, idx | nested);
        if (!nl_combi)
                goto nla_put_failure;

        nl_limits = nla_nest_start_noflag(msg, NL80211_IFACE_COMB_LIMITS |
                                               nested);
        if (!nl_limits)
                goto nla_put_failure;

        for (i = 0; i < c->n_limits; i++) {
                struct nlattr *nl_limit;

                nl_limit = nla_nest_start_noflag(msg, i + 1);
                if (!nl_limit)
                        goto nla_put_failure;
                if (nla_put_u32(msg, NL80211_IFACE_LIMIT_MAX, c->limits[i].max))
                        goto nla_put_failure;
                if (nl80211_put_iftypes(msg, NL80211_IFACE_LIMIT_TYPES,
                                        c->limits[i].types))
                        goto nla_put_failure;
                nla_nest_end(msg, nl_limit);
        }

        nla_nest_end(msg, nl_limits);

        if (c->beacon_int_infra_match &&
            nla_put_flag(msg, NL80211_IFACE_COMB_STA_AP_BI_MATCH))
                goto nla_put_failure;
        if (nla_put_u32(msg, NL80211_IFACE_COMB_NUM_CHANNELS,
                        c->num_different_channels) ||
            nla_put_u32(msg, NL80211_IFACE_COMB_MAXNUM,
                        c->max_interfaces))
                goto nla_put_failure;
        if (large &&
            (nla_put_u32(msg, NL80211_IFACE_COMB_RADAR_DETECT_WIDTHS,
                        c->radar_detect_widths) ||
             nla_put_u32(msg, NL80211_IFACE_COMB_RADAR_DETECT_REGIONS,
                        c->radar_detect_regions)))
                goto nla_put_failure;
        if (c->beacon_int_min_gcd &&
            nla_put_u32(msg, NL80211_IFACE_COMB_BI_MIN_GCD,
                        c->beacon_int_min_gcd))
                goto nla_put_failure;

        nla_nest_end(msg, nl_combi);

        return 0;
nla_put_failure:
        return -ENOBUFS;
}

static int nl80211_put_iface_combinations(struct wiphy *wiphy,
                                          struct sk_buff *msg,
                                          int attr, int radio,
                                          bool large, u16 nested)
{
        const struct ieee80211_iface_combination *c;
        struct nlattr *nl_combis;
        int i, n;

        nl_combis = nla_nest_start_noflag(msg, attr | nested);
        if (!nl_combis)
                goto nla_put_failure;

        if (radio >= 0) {
                c = wiphy->radio[0].iface_combinations;
                n = wiphy->radio[0].n_iface_combinations;
        } else {
                c = wiphy->iface_combinations;
                n = wiphy->n_iface_combinations;
        }
        for (i = 0; i < n; i++)
                if (nl80211_put_ifcomb_data(msg, large, i + 1, &c[i], nested))
                        goto nla_put_failure;

        nla_nest_end(msg, nl_combis);

        return 0;
nla_put_failure:
        return -ENOBUFS;
}

#ifdef CONFIG_PM
static int nl80211_send_wowlan_tcp_caps(struct cfg80211_registered_device *rdev,
                                        struct sk_buff *msg)
{
        const struct wiphy_wowlan_tcp_support *tcp = rdev->wiphy.wowlan->tcp;
        struct nlattr *nl_tcp;

        if (!tcp)
                return 0;

        nl_tcp = nla_nest_start_noflag(msg,
                                       NL80211_WOWLAN_TRIG_TCP_CONNECTION);
        if (!nl_tcp)
                return -ENOBUFS;

        if (nla_put_u32(msg, NL80211_WOWLAN_TCP_DATA_PAYLOAD,
                        tcp->data_payload_max))
                return -ENOBUFS;

        if (nla_put_u32(msg, NL80211_WOWLAN_TCP_DATA_PAYLOAD,
                        tcp->data_payload_max))
                return -ENOBUFS;

        if (tcp->seq && nla_put_flag(msg, NL80211_WOWLAN_TCP_DATA_PAYLOAD_SEQ))
                return -ENOBUFS;

        if (tcp->tok && nla_put(msg, NL80211_WOWLAN_TCP_DATA_PAYLOAD_TOKEN,
                                sizeof(*tcp->tok), tcp->tok))
                return -ENOBUFS;

        if (nla_put_u32(msg, NL80211_WOWLAN_TCP_DATA_INTERVAL,
                        tcp->data_interval_max))
                return -ENOBUFS;

        if (nla_put_u32(msg, NL80211_WOWLAN_TCP_WAKE_PAYLOAD,
                        tcp->wake_payload_max))
                return -ENOBUFS;

        nla_nest_end(msg, nl_tcp);
        return 0;
}

static int nl80211_send_wowlan(struct sk_buff *msg,
                               struct cfg80211_registered_device *rdev,
                               bool large)
{
        struct nlattr *nl_wowlan;

        if (!rdev->wiphy.wowlan)
                return 0;

        nl_wowlan = nla_nest_start_noflag(msg,
                                          NL80211_ATTR_WOWLAN_TRIGGERS_SUPPORTED);
        if (!nl_wowlan)
                return -ENOBUFS;

        if (((rdev->wiphy.wowlan->flags & WIPHY_WOWLAN_ANY) &&
             nla_put_flag(msg, NL80211_WOWLAN_TRIG_ANY)) ||
            ((rdev->wiphy.wowlan->flags & WIPHY_WOWLAN_DISCONNECT) &&
             nla_put_flag(msg, NL80211_WOWLAN_TRIG_DISCONNECT)) ||
            ((rdev->wiphy.wowlan->flags & WIPHY_WOWLAN_MAGIC_PKT) &&
             nla_put_flag(msg, NL80211_WOWLAN_TRIG_MAGIC_PKT)) ||
            ((rdev->wiphy.wowlan->flags & WIPHY_WOWLAN_SUPPORTS_GTK_REKEY) &&
             nla_put_flag(msg, NL80211_WOWLAN_TRIG_GTK_REKEY_SUPPORTED)) ||
            ((rdev->wiphy.wowlan->flags & WIPHY_WOWLAN_GTK_REKEY_FAILURE) &&
             nla_put_flag(msg, NL80211_WOWLAN_TRIG_GTK_REKEY_FAILURE)) ||
            ((rdev->wiphy.wowlan->flags & WIPHY_WOWLAN_EAP_IDENTITY_REQ) &&
             nla_put_flag(msg, NL80211_WOWLAN_TRIG_EAP_IDENT_REQUEST)) ||
            ((rdev->wiphy.wowlan->flags & WIPHY_WOWLAN_4WAY_HANDSHAKE) &&
             nla_put_flag(msg, NL80211_WOWLAN_TRIG_4WAY_HANDSHAKE)) ||
            ((rdev->wiphy.wowlan->flags & WIPHY_WOWLAN_RFKILL_RELEASE) &&
             nla_put_flag(msg, NL80211_WOWLAN_TRIG_RFKILL_RELEASE)))
                return -ENOBUFS;

        if (rdev->wiphy.wowlan->n_patterns) {
                struct nl80211_pattern_support pat = {
                        .max_patterns = rdev->wiphy.wowlan->n_patterns,
                        .min_pattern_len = rdev->wiphy.wowlan->pattern_min_len,
                        .max_pattern_len = rdev->wiphy.wowlan->pattern_max_len,
                        .max_pkt_offset = rdev->wiphy.wowlan->max_pkt_offset,
                };

                if (nla_put(msg, NL80211_WOWLAN_TRIG_PKT_PATTERN,
                            sizeof(pat), &pat))
                        return -ENOBUFS;
        }

        if ((rdev->wiphy.wowlan->flags & WIPHY_WOWLAN_NET_DETECT) &&
            nla_put_u32(msg, NL80211_WOWLAN_TRIG_NET_DETECT,
                        rdev->wiphy.wowlan->max_nd_match_sets))
                return -ENOBUFS;

        if (large && nl80211_send_wowlan_tcp_caps(rdev, msg))
                return -ENOBUFS;

        nla_nest_end(msg, nl_wowlan);

        return 0;
}
#endif

static int nl80211_send_coalesce(struct sk_buff *msg,
                                 struct cfg80211_registered_device *rdev)
{
        struct nl80211_coalesce_rule_support rule;

        if (!rdev->wiphy.coalesce)
                return 0;

        rule.max_rules = rdev->wiphy.coalesce->n_rules;
        rule.max_delay = rdev->wiphy.coalesce->max_delay;
        rule.pat.max_patterns = rdev->wiphy.coalesce->n_patterns;
        rule.pat.min_pattern_len = rdev->wiphy.coalesce->pattern_min_len;
        rule.pat.max_pattern_len = rdev->wiphy.coalesce->pattern_max_len;
        rule.pat.max_pkt_offset = rdev->wiphy.coalesce->max_pkt_offset;

        if (nla_put(msg, NL80211_ATTR_COALESCE_RULE, sizeof(rule), &rule))
                return -ENOBUFS;

        return 0;
}

static int
nl80211_send_iftype_data(struct sk_buff *msg,
                         const struct ieee80211_supported_band *sband,
                         const struct ieee80211_sband_iftype_data *iftdata)
{
        const struct ieee80211_sta_he_cap *he_cap = &iftdata->he_cap;
        const struct ieee80211_sta_eht_cap *eht_cap = &iftdata->eht_cap;
        const struct ieee80211_sta_uhr_cap *uhr_cap = &iftdata->uhr_cap;

        if (nl80211_put_iftypes(msg, NL80211_BAND_IFTYPE_ATTR_IFTYPES,
                                iftdata->types_mask))
                return -ENOBUFS;

        if (he_cap->has_he) {
                if (nla_put(msg, NL80211_BAND_IFTYPE_ATTR_HE_CAP_MAC,
                            sizeof(he_cap->he_cap_elem.mac_cap_info),
                            he_cap->he_cap_elem.mac_cap_info) ||
                    nla_put(msg, NL80211_BAND_IFTYPE_ATTR_HE_CAP_PHY,
                            sizeof(he_cap->he_cap_elem.phy_cap_info),
                            he_cap->he_cap_elem.phy_cap_info) ||
                    nla_put(msg, NL80211_BAND_IFTYPE_ATTR_HE_CAP_MCS_SET,
                            sizeof(he_cap->he_mcs_nss_supp),
                            &he_cap->he_mcs_nss_supp) ||
                    nla_put(msg, NL80211_BAND_IFTYPE_ATTR_HE_CAP_PPE,
                            sizeof(he_cap->ppe_thres), he_cap->ppe_thres))
                        return -ENOBUFS;
        }

        if (eht_cap->has_eht && he_cap->has_he) {
                u8 mcs_nss_size, ppe_thresh_size;
                u16 ppe_thres_hdr;
                bool is_ap;

                is_ap = iftdata->types_mask & BIT(NL80211_IFTYPE_AP) ||
                        iftdata->types_mask & BIT(NL80211_IFTYPE_P2P_GO);

                mcs_nss_size =
                        ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem,
                                                   &eht_cap->eht_cap_elem,
                                                   is_ap);

                ppe_thres_hdr = get_unaligned_le16(&eht_cap->eht_ppe_thres[0]);
                ppe_thresh_size =
                        ieee80211_eht_ppe_size(ppe_thres_hdr,
                                               eht_cap->eht_cap_elem.phy_cap_info);

                if (nla_put(msg, NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MAC,
                            sizeof(eht_cap->eht_cap_elem.mac_cap_info),
                            eht_cap->eht_cap_elem.mac_cap_info) ||
                    nla_put(msg, NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PHY,
                            sizeof(eht_cap->eht_cap_elem.phy_cap_info),
                            eht_cap->eht_cap_elem.phy_cap_info) ||
                    nla_put(msg, NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MCS_SET,
                            mcs_nss_size, &eht_cap->eht_mcs_nss_supp) ||
                    nla_put(msg, NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PPE,
                            ppe_thresh_size, eht_cap->eht_ppe_thres))
                        return -ENOBUFS;
        }

        if (uhr_cap->has_uhr) {
                if (nla_put(msg, NL80211_BAND_IFTYPE_ATTR_UHR_CAP_MAC,
                            sizeof(uhr_cap->mac), &uhr_cap->mac) ||
                    nla_put(msg, NL80211_BAND_IFTYPE_ATTR_UHR_CAP_PHY,
                            sizeof(uhr_cap->phy), &uhr_cap->phy))
                        return -ENOBUFS;
        }

        if (sband->band == NL80211_BAND_6GHZ &&
            nla_put(msg, NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA,
                    sizeof(iftdata->he_6ghz_capa),
                    &iftdata->he_6ghz_capa))
                return -ENOBUFS;

        if (iftdata->vendor_elems.data && iftdata->vendor_elems.len &&
            nla_put(msg, NL80211_BAND_IFTYPE_ATTR_VENDOR_ELEMS,
                    iftdata->vendor_elems.len, iftdata->vendor_elems.data))
                return -ENOBUFS;

        return 0;
}

static int nl80211_send_band_rateinfo(struct sk_buff *msg,
                                      struct ieee80211_supported_band *sband,
                                      bool large)
{
        struct nlattr *nl_rates, *nl_rate;
        struct ieee80211_rate *rate;
        int i;

        /* add HT info */
        if (sband->ht_cap.ht_supported &&
            (nla_put(msg, NL80211_BAND_ATTR_HT_MCS_SET,
                     sizeof(sband->ht_cap.mcs),
                     &sband->ht_cap.mcs) ||
             nla_put_u16(msg, NL80211_BAND_ATTR_HT_CAPA,
                         sband->ht_cap.cap) ||
             nla_put_u8(msg, NL80211_BAND_ATTR_HT_AMPDU_FACTOR,
                        sband->ht_cap.ampdu_factor) ||
             nla_put_u8(msg, NL80211_BAND_ATTR_HT_AMPDU_DENSITY,
                        sband->ht_cap.ampdu_density)))
                return -ENOBUFS;

        /* add VHT info */
        if (sband->vht_cap.vht_supported &&
            (nla_put(msg, NL80211_BAND_ATTR_VHT_MCS_SET,
                     sizeof(sband->vht_cap.vht_mcs),
                     &sband->vht_cap.vht_mcs) ||
             nla_put_u32(msg, NL80211_BAND_ATTR_VHT_CAPA,
                         sband->vht_cap.cap)))
                return -ENOBUFS;

        if (large && sband->n_iftype_data) {
                struct nlattr *nl_iftype_data =
                        nla_nest_start_noflag(msg,
                                              NL80211_BAND_ATTR_IFTYPE_DATA);
                const struct ieee80211_sband_iftype_data *iftd;
                int err;

                if (!nl_iftype_data)
                        return -ENOBUFS;

                for_each_sband_iftype_data(sband, i, iftd) {
                        struct nlattr *iftdata;

                        iftdata = nla_nest_start_noflag(msg, i + 1);
                        if (!iftdata)
                                return -ENOBUFS;

                        err = nl80211_send_iftype_data(msg, sband, iftd);
                        if (err)
                                return err;

                        nla_nest_end(msg, iftdata);
                }

                nla_nest_end(msg, nl_iftype_data);
        }

        /* add EDMG info */
        if (large && sband->edmg_cap.channels &&
            (nla_put_u8(msg, NL80211_BAND_ATTR_EDMG_CHANNELS,
                       sband->edmg_cap.channels) ||
            nla_put_u8(msg, NL80211_BAND_ATTR_EDMG_BW_CONFIG,
                       sband->edmg_cap.bw_config)))

                return -ENOBUFS;

        /* add bitrates */
        nl_rates = nla_nest_start_noflag(msg, NL80211_BAND_ATTR_RATES);
        if (!nl_rates)
                return -ENOBUFS;

        for (i = 0; i < sband->n_bitrates; i++) {
                nl_rate = nla_nest_start_noflag(msg, i);
                if (!nl_rate)
                        return -ENOBUFS;

                rate = &sband->bitrates[i];
                if (nla_put_u32(msg, NL80211_BITRATE_ATTR_RATE,
                                rate->bitrate))
                        return -ENOBUFS;
                if ((rate->flags & IEEE80211_RATE_SHORT_PREAMBLE) &&
                    nla_put_flag(msg,
                                 NL80211_BITRATE_ATTR_2GHZ_SHORTPREAMBLE))
                        return -ENOBUFS;

                nla_nest_end(msg, nl_rate);
        }

        nla_nest_end(msg, nl_rates);

        /* S1G capabilities */
        if (sband->band == NL80211_BAND_S1GHZ && sband->s1g_cap.s1g &&
            (nla_put(msg, NL80211_BAND_ATTR_S1G_CAPA,
                     sizeof(sband->s1g_cap.cap),
                     sband->s1g_cap.cap) ||
             nla_put(msg, NL80211_BAND_ATTR_S1G_MCS_NSS_SET,
                     sizeof(sband->s1g_cap.nss_mcs),
                     sband->s1g_cap.nss_mcs)))
                return -ENOBUFS;

        return 0;
}

static int
nl80211_send_mgmt_stypes(struct sk_buff *msg,
                         const struct ieee80211_txrx_stypes *mgmt_stypes)
{
        u16 stypes;
        struct nlattr *nl_ftypes, *nl_ifs;
        enum nl80211_iftype ift;
        int i;

        if (!mgmt_stypes)
                return 0;

        nl_ifs = nla_nest_start_noflag(msg, NL80211_ATTR_TX_FRAME_TYPES);
        if (!nl_ifs)
                return -ENOBUFS;

        for (ift = 0; ift < NUM_NL80211_IFTYPES; ift++) {
                nl_ftypes = nla_nest_start_noflag(msg, ift);
                if (!nl_ftypes)
                        return -ENOBUFS;
                i = 0;
                stypes = mgmt_stypes[ift].tx;
                while (stypes) {
                        if ((stypes & 1) &&
                            nla_put_u16(msg, NL80211_ATTR_FRAME_TYPE,
                                        (i << 4) | IEEE80211_FTYPE_MGMT))
                                return -ENOBUFS;
                        stypes >>= 1;
                        i++;
                }
                nla_nest_end(msg, nl_ftypes);
        }

        nla_nest_end(msg, nl_ifs);

        nl_ifs = nla_nest_start_noflag(msg, NL80211_ATTR_RX_FRAME_TYPES);
        if (!nl_ifs)
                return -ENOBUFS;

        for (ift = 0; ift < NUM_NL80211_IFTYPES; ift++) {
                nl_ftypes = nla_nest_start_noflag(msg, ift);
                if (!nl_ftypes)
                        return -ENOBUFS;
                i = 0;
                stypes = mgmt_stypes[ift].rx;
                while (stypes) {
                        if ((stypes & 1) &&
                            nla_put_u16(msg, NL80211_ATTR_FRAME_TYPE,
                                        (i << 4) | IEEE80211_FTYPE_MGMT))
                                return -ENOBUFS;
                        stypes >>= 1;
                        i++;
                }
                nla_nest_end(msg, nl_ftypes);
        }
        nla_nest_end(msg, nl_ifs);

        return 0;
}

#define CMD(op, n)                                                        \
         do {                                                                \
                if (rdev->ops->op) {                                        \
                        i++;                                                \
                        if (nla_put_u32(msg, i, NL80211_CMD_ ## n))         \
                                goto nla_put_failure;                        \
                }                                                        \
        } while (0)

static int nl80211_add_commands_unsplit(struct cfg80211_registered_device *rdev,
                                        struct sk_buff *msg)
{
        int i = 0;

        /*
         * do *NOT* add anything into this function, new things need to be
         * advertised only to new versions of userspace that can deal with
         * the split (and they can't possibly care about new features...
         */
        CMD(add_virtual_intf, NEW_INTERFACE);
        CMD(change_virtual_intf, SET_INTERFACE);
        CMD(add_key, NEW_KEY);
        CMD(start_ap, START_AP);
        CMD(add_station, NEW_STATION);
        CMD(add_mpath, NEW_MPATH);
        CMD(update_mesh_config, SET_MESH_CONFIG);
        CMD(change_bss, SET_BSS);
        CMD(auth, AUTHENTICATE);
        CMD(assoc, ASSOCIATE);
        CMD(deauth, DEAUTHENTICATE);
        CMD(disassoc, DISASSOCIATE);
        CMD(join_ibss, JOIN_IBSS);
        CMD(join_mesh, JOIN_MESH);
        CMD(set_pmksa, SET_PMKSA);
        CMD(del_pmksa, DEL_PMKSA);
        CMD(flush_pmksa, FLUSH_PMKSA);
        if (rdev->wiphy.flags & WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL)
                CMD(remain_on_channel, REMAIN_ON_CHANNEL);
        CMD(set_bitrate_mask, SET_TX_BITRATE_MASK);
        CMD(mgmt_tx, FRAME);
        CMD(mgmt_tx_cancel_wait, FRAME_WAIT_CANCEL);
        if (rdev->wiphy.flags & WIPHY_FLAG_NETNS_OK) {
                i++;
                if (nla_put_u32(msg, i, NL80211_CMD_SET_WIPHY_NETNS))
                        goto nla_put_failure;
        }
        if (rdev->ops->set_monitor_channel || rdev->ops->start_ap ||
            rdev->ops->join_mesh) {
                i++;
                if (nla_put_u32(msg, i, NL80211_CMD_SET_CHANNEL))
                        goto nla_put_failure;
        }
        if (rdev->wiphy.flags & WIPHY_FLAG_SUPPORTS_TDLS) {
                CMD(tdls_mgmt, TDLS_MGMT);
                CMD(tdls_oper, TDLS_OPER);
        }
        if (rdev->wiphy.max_sched_scan_reqs)
                CMD(sched_scan_start, START_SCHED_SCAN);
        CMD(probe_peer, PROBE_PEER);
        CMD(set_noack_map, SET_NOACK_MAP);
        if (rdev->wiphy.flags & WIPHY_FLAG_REPORTS_OBSS) {
                i++;
                if (nla_put_u32(msg, i, NL80211_CMD_REGISTER_BEACONS))
                        goto nla_put_failure;
        }
        CMD(start_p2p_device, START_P2P_DEVICE);
        CMD(set_mcast_rate, SET_MCAST_RATE);
#ifdef CONFIG_NL80211_TESTMODE
        CMD(testmode_cmd, TESTMODE);
#endif

        if (rdev->ops->connect || rdev->ops->auth) {
                i++;
                if (nla_put_u32(msg, i, NL80211_CMD_CONNECT))
                        goto nla_put_failure;
        }

        if (rdev->ops->disconnect || rdev->ops->deauth) {
                i++;
                if (nla_put_u32(msg, i, NL80211_CMD_DISCONNECT))
                        goto nla_put_failure;
        }

        return i;
 nla_put_failure:
        return -ENOBUFS;
}

static int
nl80211_send_pmsr_ftm_capa(const struct cfg80211_pmsr_capabilities *cap,
                           struct sk_buff *msg)
{
        struct nlattr *ftm;

        if (!cap->ftm.supported)
                return 0;

        ftm = nla_nest_start_noflag(msg, NL80211_PMSR_TYPE_FTM);
        if (!ftm)
                return -ENOBUFS;

        if (cap->ftm.asap && nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_ASAP))
                return -ENOBUFS;
        if (cap->ftm.non_asap &&
            nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_NON_ASAP))
                return -ENOBUFS;
        if (cap->ftm.request_lci &&
            nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_REQ_LCI))
                return -ENOBUFS;
        if (cap->ftm.request_civicloc &&
            nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_REQ_CIVICLOC))
                return -ENOBUFS;
        if (nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_PREAMBLES,
                        cap->ftm.preambles))
                return -ENOBUFS;
        if (nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS,
                        cap->ftm.bandwidths))
                return -ENOBUFS;
        if (cap->ftm.max_bursts_exponent >= 0 &&
            nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT,
                        cap->ftm.max_bursts_exponent))
                return -ENOBUFS;
        if (cap->ftm.max_ftms_per_burst &&
            nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST,
                        cap->ftm.max_ftms_per_burst))
                return -ENOBUFS;
        if (cap->ftm.trigger_based &&
            nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_TRIGGER_BASED))
                return -ENOBUFS;
        if (cap->ftm.non_trigger_based &&
            nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_NON_TRIGGER_BASED))
                return -ENOBUFS;
        if (cap->ftm.support_6ghz &&
            nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_6GHZ_SUPPORT))
                return -ENOBUFS;
        if (nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_TX_LTF_REP,
                        cap->ftm.max_tx_ltf_rep))
                return -ENOBUFS;
        if (nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_RX_LTF_REP,
                        cap->ftm.max_rx_ltf_rep))
                return -ENOBUFS;
        if (nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_TX_STS,
                        cap->ftm.max_tx_sts))
                return -ENOBUFS;
        if (nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_RX_STS,
                        cap->ftm.max_rx_sts))
                return -ENOBUFS;
        if (cap->ftm.max_total_ltf_tx > 0 &&
            nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_TOTAL_LTF_TX,
                        cap->ftm.max_total_ltf_tx))
                return -ENOBUFS;
        if (cap->ftm.max_total_ltf_rx > 0 &&
            nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_TOTAL_LTF_RX,
                        cap->ftm.max_total_ltf_rx))
                return -ENOBUFS;

        if (cap->ftm.ista.support_ntb || cap->ftm.ista.support_tb ||
            cap->ftm.ista.support_edca) {
                struct nlattr *ista_caps;

                ista_caps = nla_nest_start_noflag(msg,
                                                  NL80211_PMSR_FTM_CAPA_ATTR_ISTA_CAPS);
                if (!ista_caps)
                        return -ENOBUFS;
                if (cap->ftm.ista.support_ntb &&
                    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_NTB))
                        return -ENOBUFS;
                if (cap->ftm.ista.support_tb &&
                    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_TB))
                        return -ENOBUFS;
                if (cap->ftm.ista.support_edca &&
                    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_EDCA))
                        return -ENOBUFS;
                if (cap->ftm.ista.max_peers &&
                    nla_put_u32(msg, NL80211_PMSR_ATTR_MAX_PEER_ISTA_ROLE,
                                cap->ftm.ista.max_peers))
                        return -ENOBUFS;
                nla_nest_end(msg, ista_caps);
        }

        if (cap->ftm.rsta.support_ntb || cap->ftm.rsta.support_tb ||
            cap->ftm.rsta.support_edca) {
                struct nlattr *rsta_caps;

                /*
                 * Set the generic RSTA_SUPPORT flag if any of the specific
                 * ranging modes is supported to maintain the backward
                 * compatibility.
                 */
                if (nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_RSTA_SUPPORT))
                        return -ENOBUFS;

                rsta_caps = nla_nest_start_noflag(msg,
                                                  NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS);
                if (!rsta_caps)
                        return -ENOBUFS;
                if (cap->ftm.rsta.support_ntb &&
                    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_NTB))
                        return -ENOBUFS;
                if (cap->ftm.rsta.support_tb &&
                    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_TB))
                        return -ENOBUFS;
                if (cap->ftm.rsta.support_edca &&
                    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_EDCA))
                        return -ENOBUFS;
                if (cap->ftm.rsta.max_peers &&
                    nla_put_u32(msg, NL80211_PMSR_ATTR_MAX_PEER_RSTA_ROLE,
                                cap->ftm.rsta.max_peers))
                        return -ENOBUFS;
                nla_nest_end(msg, rsta_caps);
        }

        if (cap->ftm.max_no_of_tx_antennas &&
            nla_put_u8(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_TX_ANTENNAS,
                       cap->ftm.max_no_of_tx_antennas))
                return -ENOBUFS;

        if (cap->ftm.max_no_of_rx_antennas &&
            nla_put_u8(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_RX_ANTENNAS,
                       cap->ftm.max_no_of_rx_antennas))
                return -ENOBUFS;

        if (cap->ftm.min_allowed_ranging_interval_edca &&
            nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_EDCA,
                        cap->ftm.min_allowed_ranging_interval_edca))
                return -ENOBUFS;

        if (cap->ftm.min_allowed_ranging_interval_ntb &&
            nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_NTB,
                        cap->ftm.min_allowed_ranging_interval_ntb))
                return -ENOBUFS;

        if (cap->ftm.type.infra_support || cap->ftm.type.pd_support) {
                struct nlattr *pd_caps;

                pd_caps = nla_nest_start_noflag(msg,
                                                NL80211_PMSR_FTM_CAPA_ATTR_TYPE_CAPS);
                if (!pd_caps)
                        return -ENOBUFS;

                if (cap->ftm.type.infra_support &&
                    nla_put_flag(msg, NL80211_PMSR_FTM_TYPE_CAPA_ATTR_INFRA_SUPPORT))
                        return -ENOBUFS;

                if (cap->ftm.type.pd_support &&
                    nla_put_flag(msg, NL80211_PMSR_FTM_TYPE_CAPA_ATTR_PD_SUPPORT))
                        return -ENOBUFS;

                nla_nest_end(msg, pd_caps);
        }

        if (cap->ftm.concurrent_ista_rsta_support &&
            nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_CONCURRENT_ISTA_RSTA_SUPPORT))
                return -ENOBUFS;

        if (cap->ftm.type.pd_support) {
                if (cap->ftm.pd_preambles &&
                    nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_PD_PREAMBLES,
                                cap->ftm.pd_preambles))
                        return -ENOBUFS;
                if (cap->ftm.pd_bandwidths &&
                    nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_PD_BANDWIDTHS,
                                cap->ftm.pd_bandwidths))
                        return -ENOBUFS;
        }

        nla_nest_end(msg, ftm);
        return 0;
}

static int nl80211_send_pmsr_capa(struct cfg80211_registered_device *rdev,
                                  struct sk_buff *msg)
{
        const struct cfg80211_pmsr_capabilities *cap = rdev->wiphy.pmsr_capa;
        struct nlattr *pmsr, *caps;

        if (!cap)
                return 0;

        /*
         * we don't need to clean up anything here since the caller
         * will genlmsg_cancel() if we fail
         */

        pmsr = nla_nest_start_noflag(msg, NL80211_ATTR_PEER_MEASUREMENTS);
        if (!pmsr)
                return -ENOBUFS;

        if (nla_put_u32(msg, NL80211_PMSR_ATTR_MAX_PEERS, cap->max_peers))
                return -ENOBUFS;

        if (cap->report_ap_tsf &&
            nla_put_flag(msg, NL80211_PMSR_ATTR_REPORT_AP_TSF))
                return -ENOBUFS;

        if (cap->randomize_mac_addr &&
            nla_put_flag(msg, NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR))
                return -ENOBUFS;

        caps = nla_nest_start_noflag(msg, NL80211_PMSR_ATTR_TYPE_CAPA);
        if (!caps)
                return -ENOBUFS;

        if (nl80211_send_pmsr_ftm_capa(cap, msg))
                return -ENOBUFS;

        nla_nest_end(msg, caps);
        nla_nest_end(msg, pmsr);

        return 0;
}

static int
nl80211_put_iftype_akm_suites(struct cfg80211_registered_device *rdev,
                              struct sk_buff *msg)
{
        int i;
        struct nlattr *nested, *nested_akms;
        const struct wiphy_iftype_akm_suites *iftype_akms;

        if (!rdev->wiphy.num_iftype_akm_suites ||
            !rdev->wiphy.iftype_akm_suites)
                return 0;

        nested = nla_nest_start(msg, NL80211_ATTR_IFTYPE_AKM_SUITES);
        if (!nested)
                return -ENOBUFS;

        for (i = 0; i < rdev->wiphy.num_iftype_akm_suites; i++) {
                nested_akms = nla_nest_start(msg, i + 1);
                if (!nested_akms)
                        return -ENOBUFS;

                iftype_akms = &rdev->wiphy.iftype_akm_suites[i];

                if (nl80211_put_iftypes(msg, NL80211_IFTYPE_AKM_ATTR_IFTYPES,
                                        iftype_akms->iftypes_mask))
                        return -ENOBUFS;

                if (nla_put(msg, NL80211_IFTYPE_AKM_ATTR_SUITES,
                            sizeof(u32) * iftype_akms->n_akm_suites,
                            iftype_akms->akm_suites)) {
                        return -ENOBUFS;
                }
                nla_nest_end(msg, nested_akms);
        }

        nla_nest_end(msg, nested);

        return 0;
}

static int
nl80211_put_tid_config_support(struct cfg80211_registered_device *rdev,
                               struct sk_buff *msg)
{
        struct nlattr *supp;

        if (!rdev->wiphy.tid_config_support.vif &&
            !rdev->wiphy.tid_config_support.peer)
                return 0;

        supp = nla_nest_start(msg, NL80211_ATTR_TID_CONFIG);
        if (!supp)
                return -ENOSPC;

        if (rdev->wiphy.tid_config_support.vif &&
            nla_put_u64_64bit(msg, NL80211_TID_CONFIG_ATTR_VIF_SUPP,
                              rdev->wiphy.tid_config_support.vif,
                              NL80211_TID_CONFIG_ATTR_PAD))
                goto fail;

        if (rdev->wiphy.tid_config_support.peer &&
            nla_put_u64_64bit(msg, NL80211_TID_CONFIG_ATTR_PEER_SUPP,
                              rdev->wiphy.tid_config_support.peer,
                              NL80211_TID_CONFIG_ATTR_PAD))
                goto fail;

        /* for now we just use the same value ... makes more sense */
        if (nla_put_u8(msg, NL80211_TID_CONFIG_ATTR_RETRY_SHORT,
                       rdev->wiphy.tid_config_support.max_retry))
                goto fail;
        if (nla_put_u8(msg, NL80211_TID_CONFIG_ATTR_RETRY_LONG,
                       rdev->wiphy.tid_config_support.max_retry))
                goto fail;

        nla_nest_end(msg, supp);

        return 0;
fail:
        nla_nest_cancel(msg, supp);
        return -ENOBUFS;
}

static int
nl80211_put_sar_specs(struct cfg80211_registered_device *rdev,
                      struct sk_buff *msg)
{
        struct nlattr *sar_capa, *specs, *sub_freq_range;
        u8 num_freq_ranges;
        int i;

        if (!rdev->wiphy.sar_capa)
                return 0;

        num_freq_ranges = rdev->wiphy.sar_capa->num_freq_ranges;

        sar_capa = nla_nest_start(msg, NL80211_ATTR_SAR_SPEC);
        if (!sar_capa)
                return -ENOSPC;

        if (nla_put_u32(msg, NL80211_SAR_ATTR_TYPE, rdev->wiphy.sar_capa->type))
                goto fail;

        specs = nla_nest_start(msg, NL80211_SAR_ATTR_SPECS);
        if (!specs)
                goto fail;

        /* report supported freq_ranges */
        for (i = 0; i < num_freq_ranges; i++) {
                sub_freq_range = nla_nest_start(msg, i + 1);
                if (!sub_freq_range)
                        goto fail;

                if (nla_put_u32(msg, NL80211_SAR_ATTR_SPECS_START_FREQ,
                                rdev->wiphy.sar_capa->freq_ranges[i].start_freq))
                        goto fail;

                if (nla_put_u32(msg, NL80211_SAR_ATTR_SPECS_END_FREQ,
                                rdev->wiphy.sar_capa->freq_ranges[i].end_freq))
                        goto fail;

                nla_nest_end(msg, sub_freq_range);
        }

        nla_nest_end(msg, specs);
        nla_nest_end(msg, sar_capa);

        return 0;
fail:
        nla_nest_cancel(msg, sar_capa);
        return -ENOBUFS;
}

static int nl80211_put_mbssid_support(struct wiphy *wiphy, struct sk_buff *msg)
{
        struct nlattr *config;

        if (!wiphy->mbssid_max_interfaces)
                return 0;

        config = nla_nest_start(msg, NL80211_ATTR_MBSSID_CONFIG);
        if (!config)
                return -ENOBUFS;

        if (nla_put_u8(msg, NL80211_MBSSID_CONFIG_ATTR_MAX_INTERFACES,
                       wiphy->mbssid_max_interfaces))
                goto fail;

        if (wiphy->ema_max_profile_periodicity &&
            nla_put_u8(msg,
                       NL80211_MBSSID_CONFIG_ATTR_MAX_EMA_PROFILE_PERIODICITY,
                       wiphy->ema_max_profile_periodicity))
                goto fail;

        nla_nest_end(msg, config);
        return 0;

fail:
        nla_nest_cancel(msg, config);
        return -ENOBUFS;
}

static int nl80211_put_radio(struct wiphy *wiphy, struct sk_buff *msg, int idx)
{
        const struct wiphy_radio *r = &wiphy->radio[idx];
        const struct wiphy_radio_cfg *rcfg = &wiphy->radio_cfg[idx];
        struct nlattr *radio, *freq;
        int i;

        radio = nla_nest_start(msg, idx);
        if (!radio)
                return -ENOBUFS;

        if (nla_put_u32(msg, NL80211_WIPHY_RADIO_ATTR_INDEX, idx))
                goto nla_put_failure;

        if (rcfg->rts_threshold &&
            nla_put_u32(msg, NL80211_WIPHY_RADIO_ATTR_RTS_THRESHOLD,
                        rcfg->rts_threshold))
                goto nla_put_failure;

        if (r->antenna_mask &&
            nla_put_u32(msg, NL80211_WIPHY_RADIO_ATTR_ANTENNA_MASK,
                        r->antenna_mask))
                goto nla_put_failure;

        for (i = 0; i < r->n_freq_range; i++) {
                const struct wiphy_radio_freq_range *range = &r->freq_range[i];

                freq = nla_nest_start(msg, NL80211_WIPHY_RADIO_ATTR_FREQ_RANGE);
                if (!freq)
                        goto nla_put_failure;

                if (nla_put_u32(msg, NL80211_WIPHY_RADIO_FREQ_ATTR_START,
                                range->start_freq) ||
                    nla_put_u32(msg, NL80211_WIPHY_RADIO_FREQ_ATTR_END,
                                range->end_freq))
                        goto nla_put_failure;

                nla_nest_end(msg, freq);
        }

        for (i = 0; i < r->n_iface_combinations; i++)
                if (nl80211_put_ifcomb_data(msg, true,
                                            NL80211_WIPHY_RADIO_ATTR_INTERFACE_COMBINATION,
                                            &r->iface_combinations[i],
                                            NLA_F_NESTED))
                        goto nla_put_failure;

        nla_nest_end(msg, radio);

        return 0;

nla_put_failure:
        return -ENOBUFS;
}

static int nl80211_put_radios(struct wiphy *wiphy, struct sk_buff *msg)
{
        struct nlattr *radios;
        int i;

        if (!wiphy->n_radio)
                return 0;

        radios = nla_nest_start(msg, NL80211_ATTR_WIPHY_RADIOS);
        if (!radios)
                return -ENOBUFS;

        for (i = 0; i < wiphy->n_radio; i++)
                if (nl80211_put_radio(wiphy, msg, i))
                        goto fail;

        nla_nest_end(msg, radios);

        if (nl80211_put_iface_combinations(wiphy, msg,
                                           NL80211_ATTR_WIPHY_INTERFACE_COMBINATIONS,
                                           -1, true, NLA_F_NESTED))
                return -ENOBUFS;

        return 0;

fail:
        nla_nest_cancel(msg, radios);
        return -ENOBUFS;
}

static int nl80211_put_nan_phy_cap(struct wiphy *wiphy, struct sk_buff *msg)
{
        struct nlattr *nl_phy_cap;
        const struct ieee80211_sta_ht_cap *ht_cap;
        const struct ieee80211_sta_vht_cap *vht_cap;
        const struct ieee80211_sta_he_cap *he_cap;

        if (!cfg80211_iftype_allowed(wiphy, NL80211_IFTYPE_NAN_DATA, false, 0))
                return 0;

        ht_cap = &wiphy->nan_capa.phy.ht;
        vht_cap = &wiphy->nan_capa.phy.vht;
        he_cap = &wiphy->nan_capa.phy.he;

        /* HT is mandatory */
        if (WARN_ON(!ht_cap->ht_supported))
                return 0;

        nl_phy_cap = nla_nest_start_noflag(msg, NL80211_NAN_CAPA_PHY);
        if (!nl_phy_cap)
                return -ENOBUFS;

        if (nla_put(msg, NL80211_NAN_PHY_CAP_ATTR_HT_MCS_SET,
                    sizeof(ht_cap->mcs), &ht_cap->mcs) ||
            nla_put_u16(msg, NL80211_NAN_PHY_CAP_ATTR_HT_CAPA, ht_cap->cap) ||
            nla_put_u8(msg, NL80211_NAN_PHY_CAP_ATTR_HT_AMPDU_FACTOR,
                       ht_cap->ampdu_factor) ||
            nla_put_u8(msg, NL80211_NAN_PHY_CAP_ATTR_HT_AMPDU_DENSITY,
                       ht_cap->ampdu_density))
                goto fail;

        if (vht_cap->vht_supported) {
                if (nla_put(msg, NL80211_NAN_PHY_CAP_ATTR_VHT_MCS_SET,
                            sizeof(vht_cap->vht_mcs), &vht_cap->vht_mcs) ||
                    nla_put_u32(msg, NL80211_NAN_PHY_CAP_ATTR_VHT_CAPA,
                                vht_cap->cap))
                        goto fail;
        }

        if (he_cap->has_he) {
                if (nla_put(msg, NL80211_NAN_PHY_CAP_ATTR_HE_MAC,
                            sizeof(he_cap->he_cap_elem.mac_cap_info),
                            he_cap->he_cap_elem.mac_cap_info) ||
                    nla_put(msg, NL80211_NAN_PHY_CAP_ATTR_HE_PHY,
                            sizeof(he_cap->he_cap_elem.phy_cap_info),
                            he_cap->he_cap_elem.phy_cap_info) ||
                    nla_put(msg, NL80211_NAN_PHY_CAP_ATTR_HE_MCS_SET,
                            sizeof(he_cap->he_mcs_nss_supp),
                            &he_cap->he_mcs_nss_supp) ||
                    nla_put(msg, NL80211_NAN_PHY_CAP_ATTR_HE_PPE,
                            sizeof(he_cap->ppe_thres), he_cap->ppe_thres))
                        goto fail;
        }

        nla_nest_end(msg, nl_phy_cap);
        return 0;

fail:
        nla_nest_cancel(msg, nl_phy_cap);
        return -ENOBUFS;
}

static int nl80211_put_nan_capa(struct wiphy *wiphy, struct sk_buff *msg)
{
        struct nlattr *nan_caps;

        nan_caps = nla_nest_start(msg, NL80211_ATTR_NAN_CAPABILITIES);
        if (!nan_caps)
                return -ENOBUFS;

        if (wiphy->nan_capa.flags & WIPHY_NAN_FLAGS_CONFIGURABLE_SYNC &&
            nla_put_flag(msg, NL80211_NAN_CAPA_CONFIGURABLE_SYNC))
                goto fail;

        if ((wiphy->nan_capa.flags & WIPHY_NAN_FLAGS_USERSPACE_DE) &&
            nla_put_flag(msg, NL80211_NAN_CAPA_USERSPACE_DE))
                goto fail;

        if (nla_put_u8(msg, NL80211_NAN_CAPA_OP_MODE,
                       wiphy->nan_capa.op_mode) ||
            nla_put_u8(msg, NL80211_NAN_CAPA_NUM_ANTENNAS,
                       wiphy->nan_capa.n_antennas) ||
            nla_put_u16(msg, NL80211_NAN_CAPA_MAX_CHANNEL_SWITCH_TIME,
                        wiphy->nan_capa.max_channel_switch_time) ||
            nla_put_u8(msg, NL80211_NAN_CAPA_CAPABILITIES,
                       wiphy->nan_capa.dev_capabilities))
                goto fail;

        if (nl80211_put_nan_phy_cap(wiphy, msg))
                goto fail;

        nla_nest_end(msg, nan_caps);

        return 0;

fail:
        nla_nest_cancel(msg, nan_caps);
        return -ENOBUFS;
}

struct nl80211_dump_wiphy_state {
        s64 filter_wiphy;
        long start;
        long split_start, band_start, chan_start, capa_start;
        bool split;
};

static int nl80211_send_wiphy(struct cfg80211_registered_device *rdev,
                              enum nl80211_commands cmd,
                              struct sk_buff *msg, u32 portid, u32 seq,
                              int flags, struct nl80211_dump_wiphy_state *state)
{
        void *hdr;
        struct nlattr *nl_bands, *nl_band;
        struct nlattr *nl_freqs, *nl_freq;
        struct nlattr *nl_cmds;
        enum nl80211_band band;
        struct ieee80211_channel *chan;
        int i;
        const struct ieee80211_txrx_stypes *mgmt_stypes =
                                rdev->wiphy.mgmt_stypes;
        u32 features;

        hdr = nl80211hdr_put(msg, portid, seq, flags, cmd);
        if (!hdr)
                return -ENOBUFS;

        if (WARN_ON(!state))
                return -EINVAL;

        if (nla_put_u32(msg, NL80211_ATTR_WIPHY, rdev->wiphy_idx) ||
            nla_put_string(msg, NL80211_ATTR_WIPHY_NAME,
                           wiphy_name(&rdev->wiphy)) ||
            nla_put_u32(msg, NL80211_ATTR_GENERATION,
                        cfg80211_rdev_list_generation))
                goto nla_put_failure;

        if (cmd != NL80211_CMD_NEW_WIPHY)
                goto finish;

        switch (state->split_start) {
        case 0:
                if (nla_put_u8(msg, NL80211_ATTR_WIPHY_RETRY_SHORT,
                               rdev->wiphy.retry_short) ||
                    nla_put_u8(msg, NL80211_ATTR_WIPHY_RETRY_LONG,
                               rdev->wiphy.retry_long) ||
                    nla_put_u32(msg, NL80211_ATTR_WIPHY_FRAG_THRESHOLD,
                                rdev->wiphy.frag_threshold) ||
                    nla_put_u32(msg, NL80211_ATTR_WIPHY_RTS_THRESHOLD,
                                rdev->wiphy.rts_threshold) ||
                    nla_put_u8(msg, NL80211_ATTR_WIPHY_COVERAGE_CLASS,
                               rdev->wiphy.coverage_class) ||
                    nla_put_u8(msg, NL80211_ATTR_MAX_NUM_SCAN_SSIDS,
                               rdev->wiphy.max_scan_ssids) ||
                    nla_put_u8(msg, NL80211_ATTR_MAX_NUM_SCHED_SCAN_SSIDS,
                               rdev->wiphy.max_sched_scan_ssids) ||
                    nla_put_u16(msg, NL80211_ATTR_MAX_SCAN_IE_LEN,
                                rdev->wiphy.max_scan_ie_len) ||
                    nla_put_u16(msg, NL80211_ATTR_MAX_SCHED_SCAN_IE_LEN,
                                rdev->wiphy.max_sched_scan_ie_len) ||
                    nla_put_u8(msg, NL80211_ATTR_MAX_MATCH_SETS,
                               rdev->wiphy.max_match_sets))
                        goto nla_put_failure;

                if ((rdev->wiphy.flags & WIPHY_FLAG_IBSS_RSN) &&
                    nla_put_flag(msg, NL80211_ATTR_SUPPORT_IBSS_RSN))
                        goto nla_put_failure;
                if ((rdev->wiphy.flags & WIPHY_FLAG_MESH_AUTH) &&
                    nla_put_flag(msg, NL80211_ATTR_SUPPORT_MESH_AUTH))
                        goto nla_put_failure;
                if ((rdev->wiphy.flags & WIPHY_FLAG_AP_UAPSD) &&
                    nla_put_flag(msg, NL80211_ATTR_SUPPORT_AP_UAPSD))
                        goto nla_put_failure;
                if ((rdev->wiphy.flags & WIPHY_FLAG_SUPPORTS_FW_ROAM) &&
                    nla_put_flag(msg, NL80211_ATTR_ROAM_SUPPORT))
                        goto nla_put_failure;
                if ((rdev->wiphy.flags & WIPHY_FLAG_SUPPORTS_TDLS) &&
                    nla_put_flag(msg, NL80211_ATTR_TDLS_SUPPORT))
                        goto nla_put_failure;
                if ((rdev->wiphy.flags & WIPHY_FLAG_TDLS_EXTERNAL_SETUP) &&
                    nla_put_flag(msg, NL80211_ATTR_TDLS_EXTERNAL_SETUP))
                        goto nla_put_failure;
                state->split_start++;
                if (state->split)
                        break;
                fallthrough;
        case 1:
                if (nla_put(msg, NL80211_ATTR_CIPHER_SUITES,
                            sizeof(u32) * rdev->wiphy.n_cipher_suites,
                            rdev->wiphy.cipher_suites))
                        goto nla_put_failure;

                if (nla_put_u8(msg, NL80211_ATTR_MAX_NUM_PMKIDS,
                               rdev->wiphy.max_num_pmkids))
                        goto nla_put_failure;

                if ((rdev->wiphy.flags & WIPHY_FLAG_CONTROL_PORT_PROTOCOL) &&
                    nla_put_flag(msg, NL80211_ATTR_CONTROL_PORT_ETHERTYPE))
                        goto nla_put_failure;

                if (nla_put_u32(msg, NL80211_ATTR_WIPHY_ANTENNA_AVAIL_TX,
                                rdev->wiphy.available_antennas_tx) ||
                    nla_put_u32(msg, NL80211_ATTR_WIPHY_ANTENNA_AVAIL_RX,
                                rdev->wiphy.available_antennas_rx))
                        goto nla_put_failure;

                if ((rdev->wiphy.flags & WIPHY_FLAG_AP_PROBE_RESP_OFFLOAD) &&
                    nla_put_u32(msg, NL80211_ATTR_PROBE_RESP_OFFLOAD,
                                rdev->wiphy.probe_resp_offload))
                        goto nla_put_failure;

                if ((rdev->wiphy.available_antennas_tx ||
                     rdev->wiphy.available_antennas_rx) &&
                    rdev->ops->get_antenna) {
                        u32 tx_ant = 0, rx_ant = 0;
                        int res;

                        res = rdev_get_antenna(rdev, -1, &tx_ant, &rx_ant);
                        if (!res) {
                                if (nla_put_u32(msg,
                                                NL80211_ATTR_WIPHY_ANTENNA_TX,
                                                tx_ant) ||
                                    nla_put_u32(msg,
                                                NL80211_ATTR_WIPHY_ANTENNA_RX,
                                                rx_ant))
                                        goto nla_put_failure;
                        }
                }

                state->split_start++;
                if (state->split)
                        break;
                fallthrough;
        case 2:
                if (nl80211_put_iftypes(msg, NL80211_ATTR_SUPPORTED_IFTYPES,
                                        rdev->wiphy.interface_modes))
                                goto nla_put_failure;
                state->split_start++;
                if (state->split)
                        break;
                fallthrough;
        case 3:
                nl_bands = nla_nest_start_noflag(msg,
                                                 NL80211_ATTR_WIPHY_BANDS);
                if (!nl_bands)
                        goto nla_put_failure;

                for (band = state->band_start;
                     band < (state->split ?
                                NUM_NL80211_BANDS :
                                NL80211_BAND_60GHZ + 1);
                     band++) {
                        struct ieee80211_supported_band *sband;

                        /* omit higher bands for ancient software */
                        if (band > NL80211_BAND_5GHZ && !state->split)
                                break;

                        sband = rdev->wiphy.bands[band];

                        if (!sband)
                                continue;

                        nl_band = nla_nest_start_noflag(msg, band);
                        if (!nl_band)
                                goto nla_put_failure;

                        switch (state->chan_start) {
                        case 0:
                                if (nl80211_send_band_rateinfo(msg, sband,
                                                               state->split))
                                        goto nla_put_failure;
                                state->chan_start++;
                                if (state->split)
                                        break;
                                fallthrough;
                        default:
                                /* add frequencies */
                                nl_freqs = nla_nest_start_noflag(msg,
                                                                 NL80211_BAND_ATTR_FREQS);
                                if (!nl_freqs)
                                        goto nla_put_failure;

                                for (i = state->chan_start - 1;
                                     i < sband->n_channels;
                                     i++) {
                                        nl_freq = nla_nest_start_noflag(msg,
                                                                        i);
                                        if (!nl_freq)
                                                goto nla_put_failure;

                                        chan = &sband->channels[i];

                                        if (nl80211_msg_put_channel(
                                                        msg, &rdev->wiphy, chan,
                                                        state->split))
                                                goto nla_put_failure;

                                        nla_nest_end(msg, nl_freq);
                                        if (state->split)
                                                break;
                                }
                                if (i < sband->n_channels)
                                        state->chan_start = i + 2;
                                else
                                        state->chan_start = 0;
                                nla_nest_end(msg, nl_freqs);
                        }

                        nla_nest_end(msg, nl_band);

                        if (state->split) {
                                /* start again here */
                                if (state->chan_start)
                                        band--;
                                break;
                        }
                }
                nla_nest_end(msg, nl_bands);

                if (band < NUM_NL80211_BANDS)
                        state->band_start = band + 1;
                else
                        state->band_start = 0;

                /* if bands & channels are done, continue outside */
                if (state->band_start == 0 && state->chan_start == 0)
                        state->split_start++;
                if (state->split)
                        break;
                fallthrough;
        case 4:
                nl_cmds = nla_nest_start_noflag(msg,
                                                NL80211_ATTR_SUPPORTED_COMMANDS);
                if (!nl_cmds)
                        goto nla_put_failure;

                i = nl80211_add_commands_unsplit(rdev, msg);
                if (i < 0)
                        goto nla_put_failure;
                if (state->split) {
                        CMD(crit_proto_start, CRIT_PROTOCOL_START);
                        CMD(crit_proto_stop, CRIT_PROTOCOL_STOP);
                        if (rdev->wiphy.flags & WIPHY_FLAG_HAS_CHANNEL_SWITCH)
                                CMD(channel_switch, CHANNEL_SWITCH);
                        CMD(set_qos_map, SET_QOS_MAP);
                        if (rdev->wiphy.features &
                                        NL80211_FEATURE_SUPPORTS_WMM_ADMISSION)
                                CMD(add_tx_ts, ADD_TX_TS);
                        CMD(set_multicast_to_unicast, SET_MULTICAST_TO_UNICAST);
                        CMD(update_connect_params, UPDATE_CONNECT_PARAMS);
                        CMD(update_ft_ies, UPDATE_FT_IES);
                        if (rdev->wiphy.sar_capa)
                                CMD(set_sar_specs, SET_SAR_SPECS);
                        CMD(assoc_ml_reconf, ASSOC_MLO_RECONF);
                }
#undef CMD

                nla_nest_end(msg, nl_cmds);
                state->split_start++;
                if (state->split)
                        break;
                fallthrough;
        case 5:
                if (rdev->ops->remain_on_channel &&
                    (rdev->wiphy.flags & WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL) &&
                    nla_put_u32(msg,
                                NL80211_ATTR_MAX_REMAIN_ON_CHANNEL_DURATION,
                                rdev->wiphy.max_remain_on_channel_duration))
                        goto nla_put_failure;

                if ((rdev->wiphy.flags & WIPHY_FLAG_OFFCHAN_TX) &&
                    nla_put_flag(msg, NL80211_ATTR_OFFCHANNEL_TX_OK))
                        goto nla_put_failure;

                state->split_start++;
                if (state->split)
                        break;
                fallthrough;
        case 6:
#ifdef CONFIG_PM
                if (nl80211_send_wowlan(msg, rdev, state->split))
                        goto nla_put_failure;
                state->split_start++;
                if (state->split)
                        break;
#else
                state->split_start++;
#endif
                fallthrough;
        case 7:
                if (nl80211_put_iftypes(msg, NL80211_ATTR_SOFTWARE_IFTYPES,
                                        rdev->wiphy.software_iftypes))
                        goto nla_put_failure;

                if (nl80211_put_iface_combinations(&rdev->wiphy, msg,
                                                   NL80211_ATTR_INTERFACE_COMBINATIONS,
                                                   rdev->wiphy.n_radio ? 0 : -1,
                                                   state->split, 0))
                        goto nla_put_failure;

                state->split_start++;
                if (state->split)
                        break;
                fallthrough;
        case 8:
                if ((rdev->wiphy.flags & WIPHY_FLAG_HAVE_AP_SME) &&
                    nla_put_u32(msg, NL80211_ATTR_DEVICE_AP_SME,
                                rdev->wiphy.ap_sme_capa))
                        goto nla_put_failure;

                features = rdev->wiphy.features;
                /*
                 * We can only add the per-channel limit information if the
                 * dump is split, otherwise it makes it too big. Therefore
                 * only advertise it in that case.
                 */
                if (state->split)
                        features |= NL80211_FEATURE_ADVERTISE_CHAN_LIMITS;
                if (nla_put_u32(msg, NL80211_ATTR_FEATURE_FLAGS, features))
                        goto nla_put_failure;

                if (rdev->wiphy.ht_capa_mod_mask &&
                    nla_put(msg, NL80211_ATTR_HT_CAPABILITY_MASK,
                            sizeof(*rdev->wiphy.ht_capa_mod_mask),
                            rdev->wiphy.ht_capa_mod_mask))
                        goto nla_put_failure;

                if (rdev->wiphy.flags & WIPHY_FLAG_HAVE_AP_SME &&
                    rdev->wiphy.max_acl_mac_addrs &&
                    nla_put_u32(msg, NL80211_ATTR_MAC_ACL_MAX,
                                rdev->wiphy.max_acl_mac_addrs))
                        goto nla_put_failure;

                /*
                 * Any information below this point is only available to
                 * applications that can deal with it being split. This
                 * helps ensure that newly added capabilities don't break
                 * older tools by overrunning their buffers.
                 *
                 * We still increment split_start so that in the split
                 * case we'll continue with more data in the next round,
                 * but break unconditionally so unsplit data stops here.
                 */
                if (state->split)
                        state->split_start++;
                else
                        state->split_start = 0;
                break;
        case 9:
                if (nl80211_send_mgmt_stypes(msg, mgmt_stypes))
                        goto nla_put_failure;

                if (nla_put_u32(msg, NL80211_ATTR_MAX_NUM_SCHED_SCAN_PLANS,
                                rdev->wiphy.max_sched_scan_plans) ||
                    nla_put_u32(msg, NL80211_ATTR_MAX_SCAN_PLAN_INTERVAL,
                                rdev->wiphy.max_sched_scan_plan_interval) ||
                    nla_put_u32(msg, NL80211_ATTR_MAX_SCAN_PLAN_ITERATIONS,
                                rdev->wiphy.max_sched_scan_plan_iterations))
                        goto nla_put_failure;

                if (rdev->wiphy.extended_capabilities &&
                    (nla_put(msg, NL80211_ATTR_EXT_CAPA,
                             rdev->wiphy.extended_capabilities_len,
                             rdev->wiphy.extended_capabilities) ||
                     nla_put(msg, NL80211_ATTR_EXT_CAPA_MASK,
                             rdev->wiphy.extended_capabilities_len,
                             rdev->wiphy.extended_capabilities_mask)))
                        goto nla_put_failure;

                if (rdev->wiphy.vht_capa_mod_mask &&
                    nla_put(msg, NL80211_ATTR_VHT_CAPABILITY_MASK,
                            sizeof(*rdev->wiphy.vht_capa_mod_mask),
                            rdev->wiphy.vht_capa_mod_mask))
                        goto nla_put_failure;

                if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN,
                            rdev->wiphy.perm_addr))
                        goto nla_put_failure;

                if (!is_zero_ether_addr(rdev->wiphy.addr_mask) &&
                    nla_put(msg, NL80211_ATTR_MAC_MASK, ETH_ALEN,
                            rdev->wiphy.addr_mask))
                        goto nla_put_failure;

                if (rdev->wiphy.n_addresses > 1) {
                        void *attr;

                        attr = nla_nest_start(msg, NL80211_ATTR_MAC_ADDRS);
                        if (!attr)
                                goto nla_put_failure;

                        for (i = 0; i < rdev->wiphy.n_addresses; i++)
                                if (nla_put(msg, i + 1, ETH_ALEN,
                                            rdev->wiphy.addresses[i].addr))
                                        goto nla_put_failure;

                        nla_nest_end(msg, attr);
                }

                state->split_start++;
                break;
        case 10:
                if (nl80211_send_coalesce(msg, rdev))
                        goto nla_put_failure;

                if (rdev->wiphy.max_ap_assoc_sta &&
                    nla_put_u32(msg, NL80211_ATTR_MAX_AP_ASSOC_STA,
                                rdev->wiphy.max_ap_assoc_sta))
                        goto nla_put_failure;

                state->split_start++;
                break;
        case 11:
                if (rdev->wiphy.n_vendor_commands) {
                        const struct nl80211_vendor_cmd_info *info;
                        struct nlattr *nested;

                        nested = nla_nest_start_noflag(msg,
                                                       NL80211_ATTR_VENDOR_DATA);
                        if (!nested)
                                goto nla_put_failure;

                        for (i = 0; i < rdev->wiphy.n_vendor_commands; i++) {
                                info = &rdev->wiphy.vendor_commands[i].info;
                                if (nla_put(msg, i + 1, sizeof(*info), info))
                                        goto nla_put_failure;
                        }
                        nla_nest_end(msg, nested);
                }

                if (rdev->wiphy.n_vendor_events) {
                        const struct nl80211_vendor_cmd_info *info;
                        struct nlattr *nested;

                        nested = nla_nest_start_noflag(msg,
                                                       NL80211_ATTR_VENDOR_EVENTS);
                        if (!nested)
                                goto nla_put_failure;

                        for (i = 0; i < rdev->wiphy.n_vendor_events; i++) {
                                info = &rdev->wiphy.vendor_events[i];
                                if (nla_put(msg, i + 1, sizeof(*info), info))
                                        goto nla_put_failure;
                        }
                        nla_nest_end(msg, nested);
                }
                state->split_start++;
                break;
        case 12:
                if (rdev->wiphy.flags & WIPHY_FLAG_HAS_CHANNEL_SWITCH &&
                    nla_put_u8(msg, NL80211_ATTR_MAX_CSA_COUNTERS,
                               rdev->wiphy.max_num_csa_counters))
                        goto nla_put_failure;

                if (rdev->wiphy.regulatory_flags & REGULATORY_WIPHY_SELF_MANAGED &&
                    nla_put_flag(msg, NL80211_ATTR_WIPHY_SELF_MANAGED_REG))
                        goto nla_put_failure;

                if (rdev->wiphy.max_sched_scan_reqs &&
                    nla_put_u32(msg, NL80211_ATTR_SCHED_SCAN_MAX_REQS,
                                rdev->wiphy.max_sched_scan_reqs))
                        goto nla_put_failure;

                if (nla_put(msg, NL80211_ATTR_EXT_FEATURES,
                            sizeof(rdev->wiphy.ext_features),
                            rdev->wiphy.ext_features))
                        goto nla_put_failure;

                if (rdev->wiphy.bss_param_support) {
                        struct nlattr *nested;
                        u32 parsup = rdev->wiphy.bss_param_support;

                        nested = nla_nest_start(msg, NL80211_ATTR_BSS_PARAM);
                        if (!nested)
                                goto nla_put_failure;

                        if ((parsup & WIPHY_BSS_PARAM_CTS_PROT) &&
                            nla_put_flag(msg, NL80211_ATTR_BSS_CTS_PROT))
                                goto nla_put_failure;
                        if ((parsup & WIPHY_BSS_PARAM_SHORT_PREAMBLE) &&
                            nla_put_flag(msg, NL80211_ATTR_BSS_SHORT_PREAMBLE))
                                goto nla_put_failure;
                        if ((parsup & WIPHY_BSS_PARAM_SHORT_SLOT_TIME) &&
                            nla_put_flag(msg, NL80211_ATTR_BSS_SHORT_SLOT_TIME))
                                goto nla_put_failure;
                        if ((parsup & WIPHY_BSS_PARAM_BASIC_RATES) &&
                            nla_put_flag(msg, NL80211_ATTR_BSS_BASIC_RATES))
                                goto nla_put_failure;
                        if ((parsup & WIPHY_BSS_PARAM_AP_ISOLATE) &&
                            nla_put_flag(msg, NL80211_ATTR_AP_ISOLATE))
                                goto nla_put_failure;
                        if ((parsup & WIPHY_BSS_PARAM_HT_OPMODE) &&
                            nla_put_flag(msg, NL80211_ATTR_BSS_HT_OPMODE))
                                goto nla_put_failure;
                        if ((parsup & WIPHY_BSS_PARAM_P2P_CTWINDOW) &&
                            nla_put_flag(msg, NL80211_ATTR_P2P_CTWINDOW))
                                goto nla_put_failure;
                        if ((parsup & WIPHY_BSS_PARAM_P2P_OPPPS) &&
                            nla_put_flag(msg, NL80211_ATTR_P2P_OPPPS))
                                goto nla_put_failure;
                        nla_nest_end(msg, nested);
                }
                if (rdev->wiphy.bss_select_support) {
                        struct nlattr *nested;
                        u32 bss_select_support = rdev->wiphy.bss_select_support;

                        nested = nla_nest_start_noflag(msg,
                                                       NL80211_ATTR_BSS_SELECT);
                        if (!nested)
                                goto nla_put_failure;

                        i = 0;
                        while (bss_select_support) {
                                if ((bss_select_support & 1) &&
                                    nla_put_flag(msg, i))
                                        goto nla_put_failure;
                                i++;
                                bss_select_support >>= 1;
                        }
                        nla_nest_end(msg, nested);
                }

                state->split_start++;
                break;
        case 13:
                if (rdev->wiphy.num_iftype_ext_capab &&
                    rdev->wiphy.iftype_ext_capab) {
                        struct nlattr *nested_ext_capab, *nested;

                        nested = nla_nest_start_noflag(msg,
                                                       NL80211_ATTR_IFTYPE_EXT_CAPA);
                        if (!nested)
                                goto nla_put_failure;

                        for (i = state->capa_start;
                             i < rdev->wiphy.num_iftype_ext_capab; i++) {
                                const struct wiphy_iftype_ext_capab *capab;

                                capab = &rdev->wiphy.iftype_ext_capab[i];

                                nested_ext_capab = nla_nest_start_noflag(msg,
                                                                         i);
                                if (!nested_ext_capab ||
                                    nla_put_u32(msg, NL80211_ATTR_IFTYPE,
                                                capab->iftype) ||
                                    nla_put(msg, NL80211_ATTR_EXT_CAPA,
                                            capab->extended_capabilities_len,
                                            capab->extended_capabilities) ||
                                    nla_put(msg, NL80211_ATTR_EXT_CAPA_MASK,
                                            capab->extended_capabilities_len,
                                            capab->extended_capabilities_mask))
                                        goto nla_put_failure;

                                if (rdev->wiphy.flags & WIPHY_FLAG_SUPPORTS_MLO &&
                                    (nla_put_u16(msg,
                                                 NL80211_ATTR_EML_CAPABILITY,
                                                 capab->eml_capabilities) ||
                                     nla_put_u16(msg,
                                                 NL80211_ATTR_MLD_CAPA_AND_OPS,
                                                 capab->mld_capa_and_ops)))
                                        goto nla_put_failure;
                                if (rdev->wiphy.flags & WIPHY_FLAG_SUPPORTS_MLO &&
                                    capab->ext_mld_capa_and_ops &&
                                    nla_put_u16(msg,
                                                NL80211_ATTR_EXT_MLD_CAPA_AND_OPS,
                                                capab->ext_mld_capa_and_ops))
                                        goto nla_put_failure;

                                nla_nest_end(msg, nested_ext_capab);
                                if (state->split)
                                        break;
                        }
                        nla_nest_end(msg, nested);
                        if (i < rdev->wiphy.num_iftype_ext_capab) {
                                state->capa_start = i + 1;
                                break;
                        }
                }

                if (nla_put_u32(msg, NL80211_ATTR_BANDS,
                                rdev->wiphy.nan_supported_bands))
                        goto nla_put_failure;

                if (wiphy_ext_feature_isset(&rdev->wiphy,
                                            NL80211_EXT_FEATURE_TXQS)) {
                        struct cfg80211_txq_stats txqstats = {};
                        int res;

                        res = rdev_get_txq_stats(rdev, NULL, &txqstats);
                        if (!res &&
                            !nl80211_put_txq_stats(msg, &txqstats,
                                                   NL80211_ATTR_TXQ_STATS))
                                goto nla_put_failure;

                        if (nla_put_u32(msg, NL80211_ATTR_TXQ_LIMIT,
                                        rdev->wiphy.txq_limit))
                                goto nla_put_failure;
                        if (nla_put_u32(msg, NL80211_ATTR_TXQ_MEMORY_LIMIT,
                                        rdev->wiphy.txq_memory_limit))
                                goto nla_put_failure;
                        if (nla_put_u32(msg, NL80211_ATTR_TXQ_QUANTUM,
                                        rdev->wiphy.txq_quantum))
                                goto nla_put_failure;
                }

                state->split_start++;
                break;
        case 14:
                if (nl80211_send_pmsr_capa(rdev, msg))
                        goto nla_put_failure;

                state->split_start++;
                break;
        case 15:
                if (rdev->wiphy.akm_suites &&
                    nla_put(msg, NL80211_ATTR_AKM_SUITES,
                            sizeof(u32) * rdev->wiphy.n_akm_suites,
                            rdev->wiphy.akm_suites))
                        goto nla_put_failure;

                if (nl80211_put_iftype_akm_suites(rdev, msg))
                        goto nla_put_failure;

                if (nl80211_put_tid_config_support(rdev, msg))
                        goto nla_put_failure;
                state->split_start++;
                break;
        case 16:
                if (nl80211_put_sar_specs(rdev, msg))
                        goto nla_put_failure;

                if (nl80211_put_mbssid_support(&rdev->wiphy, msg))
                        goto nla_put_failure;

                if (nla_put_u16(msg, NL80211_ATTR_MAX_NUM_AKM_SUITES,
                                rdev->wiphy.max_num_akm_suites))
                        goto nla_put_failure;

                if (rdev->wiphy.flags & WIPHY_FLAG_SUPPORTS_MLO)
                        nla_put_flag(msg, NL80211_ATTR_MLO_SUPPORT);

                if (rdev->wiphy.hw_timestamp_max_peers &&
                    nla_put_u16(msg, NL80211_ATTR_MAX_HW_TIMESTAMP_PEERS,
                                rdev->wiphy.hw_timestamp_max_peers))
                        goto nla_put_failure;

                state->split_start++;
                break;
        case 17:
                if (nl80211_put_radios(&rdev->wiphy, msg))
                        goto nla_put_failure;

                state->split_start++;
                break;
        case 18:
                if (nl80211_put_nan_capa(&rdev->wiphy, msg))
                        goto nla_put_failure;

                /* done */
                state->split_start = 0;
                break;
        }
 finish:
        genlmsg_end(msg, hdr);
        return 0;

 nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

static int nl80211_dump_wiphy_parse(struct sk_buff *skb,
                                    struct netlink_callback *cb,
                                    struct nl80211_dump_wiphy_state *state)
{
        struct nlattr **tb = kzalloc_objs(*tb, NUM_NL80211_ATTR);
        int ret;

        if (!tb)
                return -ENOMEM;

        ret = nlmsg_parse_deprecated(cb->nlh,
                                     GENL_HDRLEN + nl80211_fam.hdrsize,
                                     tb, nl80211_fam.maxattr,
                                     nl80211_policy, NULL);
        /* ignore parse errors for backward compatibility */
        if (ret) {
                ret = 0;
                goto out;
        }

        state->split = tb[NL80211_ATTR_SPLIT_WIPHY_DUMP];
        if (tb[NL80211_ATTR_WIPHY])
                state->filter_wiphy = nla_get_u32(tb[NL80211_ATTR_WIPHY]);
        if (tb[NL80211_ATTR_WDEV])
                state->filter_wiphy = nla_get_u64(tb[NL80211_ATTR_WDEV]) >> 32;
        if (tb[NL80211_ATTR_IFINDEX]) {
                struct net_device *netdev;
                struct cfg80211_registered_device *rdev;
                int ifidx = nla_get_u32(tb[NL80211_ATTR_IFINDEX]);

                netdev = __dev_get_by_index(sock_net(skb->sk), ifidx);
                if (!netdev) {
                        ret = -ENODEV;
                        goto out;
                }
                if (netdev->ieee80211_ptr) {
                        rdev = wiphy_to_rdev(
                                netdev->ieee80211_ptr->wiphy);
                        state->filter_wiphy = rdev->wiphy_idx;
                }
        }

        ret = 0;
out:
        kfree(tb);
        return ret;
}

static int nl80211_dump_wiphy(struct sk_buff *skb, struct netlink_callback *cb)
{
        int idx = 0, ret;
        struct nl80211_dump_wiphy_state *state = (void *)cb->args[0];
        struct cfg80211_registered_device *rdev;

        rtnl_lock();
        if (!state) {
                state = kzalloc_obj(*state);
                if (!state) {
                        rtnl_unlock();
                        return -ENOMEM;
                }
                state->filter_wiphy = -1;
                ret = nl80211_dump_wiphy_parse(skb, cb, state);
                if (ret) {
                        kfree(state);
                        rtnl_unlock();
                        return ret;
                }
                cb->args[0] = (long)state;
        }

        for_each_rdev(rdev) {
                if (!net_eq(wiphy_net(&rdev->wiphy), sock_net(skb->sk)))
                        continue;
                if (++idx <= state->start)
                        continue;
                if (state->filter_wiphy != -1 &&
                    state->filter_wiphy != rdev->wiphy_idx)
                        continue;
                wiphy_lock(&rdev->wiphy);
                /* attempt to fit multiple wiphy data chunks into the skb */
                do {
                        ret = nl80211_send_wiphy(rdev, NL80211_CMD_NEW_WIPHY,
                                                 skb,
                                                 NETLINK_CB(cb->skb).portid,
                                                 cb->nlh->nlmsg_seq,
                                                 NLM_F_MULTI, state);
                        if (ret < 0) {
                                /*
                                 * If sending the wiphy data didn't fit (ENOBUFS
                                 * or EMSGSIZE returned), this SKB is still
                                 * empty (so it's not too big because another
                                 * wiphy dataset is already in the skb) and
                                 * we've not tried to adjust the dump allocation
                                 * yet ... then adjust the alloc size to be
                                 * bigger, and return 1 but with the empty skb.
                                 * This results in an empty message being RX'ed
                                 * in userspace, but that is ignored.
                                 *
                                 * We can then retry with the larger buffer.
                                 */
                                if ((ret == -ENOBUFS || ret == -EMSGSIZE) &&
                                    !skb->len && !state->split &&
                                    cb->min_dump_alloc < 4096) {
                                        cb->min_dump_alloc = 4096;
                                        state->split_start = 0;
                                        wiphy_unlock(&rdev->wiphy);
                                        rtnl_unlock();
                                        return 1;
                                }
                                idx--;
                                break;
                        }
                } while (state->split_start > 0);
                wiphy_unlock(&rdev->wiphy);
                break;
        }
        rtnl_unlock();

        state->start = idx;

        return skb->len;
}

static int nl80211_dump_wiphy_done(struct netlink_callback *cb)
{
        kfree((void *)cb->args[0]);
        return 0;
}

static int nl80211_get_wiphy(struct sk_buff *skb, struct genl_info *info)
{
        struct sk_buff *msg;
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct nl80211_dump_wiphy_state state = {};

        msg = nlmsg_new(4096, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        if (nl80211_send_wiphy(rdev, NL80211_CMD_NEW_WIPHY, msg,
                               info->snd_portid, info->snd_seq, 0,
                               &state) < 0) {
                nlmsg_free(msg);
                return -ENOBUFS;
        }

        return genlmsg_reply(msg, info);
}

static const struct nla_policy txq_params_policy[NL80211_TXQ_ATTR_MAX + 1] = {
        [NL80211_TXQ_ATTR_QUEUE]                = { .type = NLA_U8 },
        [NL80211_TXQ_ATTR_TXOP]                        = { .type = NLA_U16 },
        [NL80211_TXQ_ATTR_CWMIN]                = { .type = NLA_U16 },
        [NL80211_TXQ_ATTR_CWMAX]                = { .type = NLA_U16 },
        [NL80211_TXQ_ATTR_AIFS]                        = { .type = NLA_U8 },
};

static int parse_txq_params(struct nlattr *tb[],
                            struct ieee80211_txq_params *txq_params)
{
        u8 ac;

        if (!tb[NL80211_TXQ_ATTR_AC] || !tb[NL80211_TXQ_ATTR_TXOP] ||
            !tb[NL80211_TXQ_ATTR_CWMIN] || !tb[NL80211_TXQ_ATTR_CWMAX] ||
            !tb[NL80211_TXQ_ATTR_AIFS])
                return -EINVAL;

        ac = nla_get_u8(tb[NL80211_TXQ_ATTR_AC]);
        txq_params->txop = nla_get_u16(tb[NL80211_TXQ_ATTR_TXOP]);
        txq_params->cwmin = nla_get_u16(tb[NL80211_TXQ_ATTR_CWMIN]);
        txq_params->cwmax = nla_get_u16(tb[NL80211_TXQ_ATTR_CWMAX]);
        txq_params->aifs = nla_get_u8(tb[NL80211_TXQ_ATTR_AIFS]);

        if (ac >= NL80211_NUM_ACS)
                return -EINVAL;
        txq_params->ac = array_index_nospec(ac, NL80211_NUM_ACS);
        return 0;
}

static bool nl80211_can_set_dev_channel(struct wireless_dev *wdev)
{
        /*
         * You can only set the channel explicitly for some interfaces,
         * most have their channel managed via their respective
         * "establish a connection" command (connect, join, ...)
         *
         * For AP/GO and mesh mode, the channel can be set with the
         * channel userspace API, but is only stored and passed to the
         * low-level driver when the AP starts or the mesh is joined.
         * This is for backward compatibility, userspace can also give
         * the channel in the start-ap or join-mesh commands instead.
         *
         * Monitors are special as they are normally slaved to
         * whatever else is going on, so they have their own special
         * operation to set the monitor channel if possible.
         */
        return !wdev ||
                wdev->iftype == NL80211_IFTYPE_AP ||
                wdev->iftype == NL80211_IFTYPE_MESH_POINT ||
                wdev->iftype == NL80211_IFTYPE_MONITOR ||
                wdev->iftype == NL80211_IFTYPE_P2P_GO;
}

static int _nl80211_parse_chandef(struct cfg80211_registered_device *rdev,
                                  struct netlink_ext_ack *extack,
                                  struct nlattr **attrs, bool monitor,
                                  struct cfg80211_chan_def *chandef,
                                  bool permit_npca)
{
        u32 control_freq;

        if (!attrs[NL80211_ATTR_WIPHY_FREQ]) {
                NL_SET_ERR_MSG_ATTR(extack, attrs[NL80211_ATTR_WIPHY_FREQ],
                                    "Frequency is missing");
                return -EINVAL;
        }

        control_freq = MHZ_TO_KHZ(
                        nla_get_u32(attrs[NL80211_ATTR_WIPHY_FREQ]));
        if (attrs[NL80211_ATTR_WIPHY_FREQ_OFFSET])
                control_freq +=
                    nla_get_u32(attrs[NL80211_ATTR_WIPHY_FREQ_OFFSET]);

        memset(chandef, 0, sizeof(*chandef));
        chandef->chan = ieee80211_get_channel_khz(&rdev->wiphy, control_freq);
        chandef->width = NL80211_CHAN_WIDTH_20_NOHT;
        chandef->center_freq1 = KHZ_TO_MHZ(control_freq);
        chandef->freq1_offset = control_freq % 1000;
        chandef->center_freq2 = 0;
        chandef->s1g_primary_2mhz = false;

        if (!chandef->chan) {
                NL_SET_ERR_MSG_ATTR(extack, attrs[NL80211_ATTR_WIPHY_FREQ],
                                    "Unknown channel");
                return -EINVAL;
        }

        if (cfg80211_chandef_is_s1g(chandef))
                chandef->width = NL80211_CHAN_WIDTH_1;

        if (attrs[NL80211_ATTR_WIPHY_CHANNEL_TYPE]) {
                enum nl80211_channel_type chantype;

                chantype = nla_get_u32(attrs[NL80211_ATTR_WIPHY_CHANNEL_TYPE]);

                switch (chantype) {
                case NL80211_CHAN_NO_HT:
                case NL80211_CHAN_HT20:
                case NL80211_CHAN_HT40PLUS:
                case NL80211_CHAN_HT40MINUS:
                        if (chandef->chan->band == NL80211_BAND_60GHZ ||
                            chandef->chan->band == NL80211_BAND_S1GHZ)
                                return -EINVAL;
                        cfg80211_chandef_create(chandef, chandef->chan,
                                                chantype);
                        /* user input for center_freq is incorrect */
                        if (attrs[NL80211_ATTR_CENTER_FREQ1] &&
                            chandef->center_freq1 != nla_get_u32(attrs[NL80211_ATTR_CENTER_FREQ1])) {
                                NL_SET_ERR_MSG_ATTR(extack,
                                                    attrs[NL80211_ATTR_CENTER_FREQ1],
                                                    "bad center frequency 1");
                                return -EINVAL;
                        }
                        /* center_freq2 must be zero */
                        if (attrs[NL80211_ATTR_CENTER_FREQ2] &&
                            nla_get_u32(attrs[NL80211_ATTR_CENTER_FREQ2])) {
                                NL_SET_ERR_MSG_ATTR(extack,
                                                    attrs[NL80211_ATTR_CENTER_FREQ2],
                                                    "center frequency 2 can't be used");
                                return -EINVAL;
                        }
                        break;
                default:
                        NL_SET_ERR_MSG_ATTR(extack,
                                            attrs[NL80211_ATTR_WIPHY_CHANNEL_TYPE],
                                            "invalid channel type");
                        return -EINVAL;
                }
        } else if (attrs[NL80211_ATTR_CHANNEL_WIDTH]) {
                chandef->width = nla_get_u32(attrs[NL80211_ATTR_CHANNEL_WIDTH]);
                if (attrs[NL80211_ATTR_CENTER_FREQ1]) {
                        chandef->center_freq1 =
                                nla_get_u32(attrs[NL80211_ATTR_CENTER_FREQ1]);
                        chandef->freq1_offset = nla_get_u32_default(
                                attrs[NL80211_ATTR_CENTER_FREQ1_OFFSET], 0);
                }

                if (attrs[NL80211_ATTR_CENTER_FREQ2])
                        chandef->center_freq2 =
                                nla_get_u32(attrs[NL80211_ATTR_CENTER_FREQ2]);

                chandef->s1g_primary_2mhz = nla_get_flag(
                        attrs[NL80211_ATTR_S1G_PRIMARY_2MHZ]);
        }

        if (attrs[NL80211_ATTR_WIPHY_EDMG_CHANNELS]) {
                chandef->edmg.channels =
                      nla_get_u8(attrs[NL80211_ATTR_WIPHY_EDMG_CHANNELS]);

                if (attrs[NL80211_ATTR_WIPHY_EDMG_BW_CONFIG])
                        chandef->edmg.bw_config =
                     nla_get_u8(attrs[NL80211_ATTR_WIPHY_EDMG_BW_CONFIG]);
        } else {
                chandef->edmg.bw_config = 0;
                chandef->edmg.channels = 0;
        }

        if (attrs[NL80211_ATTR_PUNCT_BITMAP]) {
                chandef->punctured =
                        nla_get_u32(attrs[NL80211_ATTR_PUNCT_BITMAP]);

                if (chandef->punctured &&
                    !wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_PUNCT)) {
                        NL_SET_ERR_MSG_ATTR(extack,
                                            attrs[NL80211_ATTR_WIPHY_FREQ],
                                            "driver doesn't support puncturing");
                        return -EINVAL;
                }
        }

        if (attrs[NL80211_ATTR_NPCA_PRIMARY_FREQ]) {
                if (!permit_npca) {
                        NL_SET_ERR_MSG_ATTR(extack,
                                            attrs[NL80211_ATTR_NPCA_PRIMARY_FREQ],
                                            "NPCA not supported");
                        return -EINVAL;
                }

                chandef->npca_chan =
                        ieee80211_get_channel(&rdev->wiphy,
                                              nla_get_u32(attrs[NL80211_ATTR_NPCA_PRIMARY_FREQ]));
                if (!chandef->npca_chan) {
                        NL_SET_ERR_MSG_ATTR(extack,
                                            attrs[NL80211_ATTR_NPCA_PRIMARY_FREQ],
                                            "invalid NPCA primary channel");
                        return -EINVAL;
                }

                chandef->npca_punctured =
                        nla_get_u32_default(attrs[NL80211_ATTR_NPCA_PUNCT_BITMAP],
                                            chandef->punctured);
        } else if (attrs[NL80211_ATTR_NPCA_PUNCT_BITMAP]) {
                NL_SET_ERR_MSG_ATTR(extack,
                                    attrs[NL80211_ATTR_NPCA_PUNCT_BITMAP],
                                    "NPCA puncturing only valid with NPCA");
                return -EINVAL;
        }

        if (!cfg80211_chandef_valid(chandef)) {
                NL_SET_ERR_MSG_ATTR(extack, attrs[NL80211_ATTR_WIPHY_FREQ],
                                    "invalid channel definition");
                return -EINVAL;
        }

        if (!_cfg80211_chandef_usable(&rdev->wiphy, chandef,
                                      IEEE80211_CHAN_DISABLED,
                                      monitor ? IEEE80211_CHAN_CAN_MONITOR : 0)) {
                NL_SET_ERR_MSG_ATTR(extack, attrs[NL80211_ATTR_WIPHY_FREQ],
                                    "(extension) channel is disabled");
                return -EINVAL;
        }

        if (chandef->width == NL80211_CHAN_WIDTH_5 ||
            chandef->width == NL80211_CHAN_WIDTH_10) {
                NL_SET_ERR_MSG(extack, "5/10 MHz not supported any more");
                return -EINVAL;
        }

        return 0;
}

int nl80211_parse_chandef(struct cfg80211_registered_device *rdev,
                          struct netlink_ext_ack *extack,
                          struct nlattr **attrs,
                          struct cfg80211_chan_def *chandef,
                          bool permit_npca)
{
        return _nl80211_parse_chandef(rdev, extack, attrs, false, chandef,
                                      permit_npca);
}

static int __nl80211_set_channel(struct cfg80211_registered_device *rdev,
                                 struct net_device *dev,
                                 struct genl_info *info,
                                 int _link_id)
{
        struct cfg80211_chan_def chandef;
        int result;
        enum nl80211_iftype iftype = NL80211_IFTYPE_MONITOR;
        struct wireless_dev *wdev = NULL;
        int link_id = _link_id;
        bool permit_npca;

        if (dev)
                wdev = dev->ieee80211_ptr;
        if (!nl80211_can_set_dev_channel(wdev))
                return -EOPNOTSUPP;
        if (wdev)
                iftype = wdev->iftype;

        if (link_id < 0) {
                if (wdev && wdev->valid_links)
                        return -EINVAL;
                link_id = 0;
        }

        /* allow parsing it - will check on start_ap or below */
        permit_npca = iftype == NL80211_IFTYPE_AP ||
                      iftype == NL80211_IFTYPE_P2P_GO;

        result = _nl80211_parse_chandef(rdev, info->extack, info->attrs,
                                        iftype == NL80211_IFTYPE_MONITOR,
                                        &chandef, permit_npca);
        if (result)
                return result;

        switch (iftype) {
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_P2P_GO:
                if (!cfg80211_reg_can_beacon_relax(&rdev->wiphy, &chandef,
                                                   iftype))
                        return -EINVAL;
                if (wdev->links[link_id].ap.beacon_interval) {
                        struct ieee80211_channel *cur_chan;

                        if (!dev || !rdev->ops->set_ap_chanwidth ||
                            !(rdev->wiphy.features &
                              NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE))
                                return -EBUSY;

                        /* Only allow dynamic channel width changes */
                        cur_chan = wdev->links[link_id].ap.chandef.npca_chan;
                        if (chandef.npca_chan != cur_chan)
                                return -EBUSY;
                        cur_chan = wdev->links[link_id].ap.chandef.chan;
                        if (chandef.chan != cur_chan)
                                return -EBUSY;

                        /* only allow this for regular channel widths */
                        switch (wdev->links[link_id].ap.chandef.width) {
                        case NL80211_CHAN_WIDTH_20_NOHT:
                        case NL80211_CHAN_WIDTH_20:
                        case NL80211_CHAN_WIDTH_40:
                        case NL80211_CHAN_WIDTH_80:
                        case NL80211_CHAN_WIDTH_80P80:
                        case NL80211_CHAN_WIDTH_160:
                        case NL80211_CHAN_WIDTH_320:
                                break;
                        default:
                                return -EINVAL;
                        }

                        switch (chandef.width) {
                        case NL80211_CHAN_WIDTH_20_NOHT:
                        case NL80211_CHAN_WIDTH_20:
                        case NL80211_CHAN_WIDTH_40:
                        case NL80211_CHAN_WIDTH_80:
                        case NL80211_CHAN_WIDTH_80P80:
                        case NL80211_CHAN_WIDTH_160:
                        case NL80211_CHAN_WIDTH_320:
                                break;
                        default:
                                return -EINVAL;
                        }

                        result = rdev_set_ap_chanwidth(rdev, dev, link_id,
                                                       &chandef);
                        if (result)
                                return result;
                        wdev->links[link_id].ap.chandef = chandef;
                } else {
                        wdev->u.ap.preset_chandef = chandef;
                }
                return 0;
        case NL80211_IFTYPE_MESH_POINT:
                return cfg80211_set_mesh_channel(rdev, wdev, &chandef);
        case NL80211_IFTYPE_MONITOR:
                return cfg80211_set_monitor_channel(rdev, dev, &chandef);
        default:
                break;
        }

        return -EINVAL;
}

static int nl80211_set_channel(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        int link_id = nl80211_link_id_or_invalid(info->attrs);
        struct net_device *netdev = info->user_ptr[1];

        return __nl80211_set_channel(rdev, netdev, info, link_id);
}

static int nl80211_set_wiphy_radio(struct genl_info *info,
                                   struct cfg80211_registered_device *rdev,
                                   int radio_idx)
{
        u32 rts_threshold = 0, old_rts, changed = 0;
        int result;

        if (!rdev->ops->set_wiphy_params)
                return -EOPNOTSUPP;

        if (info->attrs[NL80211_ATTR_WIPHY_RTS_THRESHOLD]) {
                rts_threshold = nla_get_u32(
                                info->attrs[NL80211_ATTR_WIPHY_RTS_THRESHOLD]);
                changed |= WIPHY_PARAM_RTS_THRESHOLD;
        }

        old_rts = rdev->wiphy.radio_cfg[radio_idx].rts_threshold;

        rdev->wiphy.radio_cfg[radio_idx].rts_threshold = rts_threshold;

        result = rdev_set_wiphy_params(rdev, radio_idx, changed);
        if (result)
                rdev->wiphy.radio_cfg[radio_idx].rts_threshold = old_rts;

        return 0;
}

static int nl80211_set_wiphy(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = NULL;
        struct net_device *netdev = NULL;
        struct wireless_dev *wdev;
        int result = 0, rem_txq_params = 0;
        struct nlattr *nl_txq_params;
        u32 changed;
        u8 retry_short = 0, retry_long = 0;
        u32 frag_threshold = 0, rts_threshold = 0;
        u8 coverage_class = 0;
        u32 txq_limit = 0, txq_memory_limit = 0, txq_quantum = 0;
        int radio_idx = -1;

        rtnl_lock();
        /*
         * Try to find the wiphy and netdev. Normally this
         * function shouldn't need the netdev, but this is
         * done for backward compatibility -- previously
         * setting the channel was done per wiphy, but now
         * it is per netdev. Previous userland like hostapd
         * also passed a netdev to set_wiphy, so that it is
         * possible to let that go to the right netdev!
         */

        if (info->attrs[NL80211_ATTR_IFINDEX]) {
                int ifindex = nla_get_u32(info->attrs[NL80211_ATTR_IFINDEX]);

                netdev = __dev_get_by_index(genl_info_net(info), ifindex);
                if (netdev && netdev->ieee80211_ptr)
                        rdev = wiphy_to_rdev(netdev->ieee80211_ptr->wiphy);
                else
                        netdev = NULL;
        }

        if (!netdev) {
                rdev = __cfg80211_rdev_from_attrs(genl_info_net(info),
                                                  info->attrs);
                if (IS_ERR(rdev)) {
                        rtnl_unlock();
                        return PTR_ERR(rdev);
                }
                wdev = NULL;
                netdev = NULL;
                result = 0;
        } else
                wdev = netdev->ieee80211_ptr;

        guard(wiphy)(&rdev->wiphy);

        /*
         * end workaround code, by now the rdev is available
         * and locked, and wdev may or may not be NULL.
         */

        if (info->attrs[NL80211_ATTR_WIPHY_NAME])
                result = cfg80211_dev_rename(
                        rdev, nla_data(info->attrs[NL80211_ATTR_WIPHY_NAME]));
        rtnl_unlock();

        if (result)
                return result;

        if (info->attrs[NL80211_ATTR_WIPHY_RADIO_INDEX]) {
                /* Radio idx is not expected for non-multi radio wiphy */
                if (rdev->wiphy.n_radio <= 0)
                        return -EINVAL;

                radio_idx = nla_get_u8(
                                info->attrs[NL80211_ATTR_WIPHY_RADIO_INDEX]);
                if (radio_idx >= rdev->wiphy.n_radio)
                        return -EINVAL;

                return nl80211_set_wiphy_radio(info, rdev, radio_idx);
        }

        if (info->attrs[NL80211_ATTR_WIPHY_TXQ_PARAMS]) {
                struct ieee80211_txq_params txq_params;
                struct nlattr *tb[NL80211_TXQ_ATTR_MAX + 1];

                if (!rdev->ops->set_txq_params)
                        return -EOPNOTSUPP;

                if (!netdev)
                        return -EINVAL;

                if (netdev->ieee80211_ptr->iftype != NL80211_IFTYPE_AP &&
                    netdev->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_GO)
                        return -EINVAL;

                if (!netif_running(netdev))
                        return -ENETDOWN;

                nla_for_each_nested(nl_txq_params,
                                    info->attrs[NL80211_ATTR_WIPHY_TXQ_PARAMS],
                                    rem_txq_params) {
                        result = nla_parse_nested_deprecated(tb,
                                                             NL80211_TXQ_ATTR_MAX,
                                                             nl_txq_params,
                                                             txq_params_policy,
                                                             info->extack);
                        if (result)
                                return result;

                        result = parse_txq_params(tb, &txq_params);
                        if (result)
                                return result;

                        txq_params.link_id =
                                nl80211_link_id_or_invalid(info->attrs);

                        if (txq_params.link_id >= 0 &&
                            !(netdev->ieee80211_ptr->valid_links &
                              BIT(txq_params.link_id)))
                                result = -ENOLINK;
                        else if (txq_params.link_id >= 0 &&
                                 !netdev->ieee80211_ptr->valid_links)
                                result = -EINVAL;
                        else
                                result = rdev_set_txq_params(rdev, netdev,
                                                             &txq_params);
                        if (result)
                                return result;
                }
        }

        if (info->attrs[NL80211_ATTR_WIPHY_FREQ]) {
                int link_id = nl80211_link_id_or_invalid(info->attrs);

                if (wdev) {
                        result = __nl80211_set_channel(
                                rdev,
                                nl80211_can_set_dev_channel(wdev) ? netdev : NULL,
                                info, link_id);
                } else {
                        result = __nl80211_set_channel(rdev, netdev, info, link_id);
                }

                if (result)
                        return result;
        }

        if (info->attrs[NL80211_ATTR_WIPHY_TX_POWER_SETTING]) {
                struct wireless_dev *txp_wdev = wdev;
                enum nl80211_tx_power_setting type;
                int idx, mbm = 0;

                if (!(rdev->wiphy.features & NL80211_FEATURE_VIF_TXPOWER))
                        txp_wdev = NULL;

                if (!rdev->ops->set_tx_power)
                        return -EOPNOTSUPP;

                idx = NL80211_ATTR_WIPHY_TX_POWER_SETTING;
                type = nla_get_u32(info->attrs[idx]);

                if (!info->attrs[NL80211_ATTR_WIPHY_TX_POWER_LEVEL] &&
                    (type != NL80211_TX_POWER_AUTOMATIC))
                        return -EINVAL;

                if (type != NL80211_TX_POWER_AUTOMATIC) {
                        idx = NL80211_ATTR_WIPHY_TX_POWER_LEVEL;
                        mbm = nla_get_u32(info->attrs[idx]);
                }

                result = rdev_set_tx_power(rdev, txp_wdev, radio_idx, type,
                                           mbm);
                if (result)
                        return result;
        }

        if (info->attrs[NL80211_ATTR_WIPHY_ANTENNA_TX] &&
            info->attrs[NL80211_ATTR_WIPHY_ANTENNA_RX]) {
                u32 tx_ant, rx_ant;

                if ((!rdev->wiphy.available_antennas_tx &&
                     !rdev->wiphy.available_antennas_rx) ||
                    !rdev->ops->set_antenna)
                        return -EOPNOTSUPP;

                tx_ant = nla_get_u32(info->attrs[NL80211_ATTR_WIPHY_ANTENNA_TX]);
                rx_ant = nla_get_u32(info->attrs[NL80211_ATTR_WIPHY_ANTENNA_RX]);

                /* reject antenna configurations which don't match the
                 * available antenna masks, except for the "all" mask */
                if ((~tx_ant && (tx_ant & ~rdev->wiphy.available_antennas_tx)) ||
                    (~rx_ant && (rx_ant & ~rdev->wiphy.available_antennas_rx)))
                        return -EINVAL;

                tx_ant = tx_ant & rdev->wiphy.available_antennas_tx;
                rx_ant = rx_ant & rdev->wiphy.available_antennas_rx;

                result = rdev_set_antenna(rdev, radio_idx, tx_ant, rx_ant);
                if (result)
                        return result;
        }

        changed = 0;

        if (info->attrs[NL80211_ATTR_WIPHY_RETRY_SHORT]) {
                retry_short = nla_get_u8(
                        info->attrs[NL80211_ATTR_WIPHY_RETRY_SHORT]);

                changed |= WIPHY_PARAM_RETRY_SHORT;
        }

        if (info->attrs[NL80211_ATTR_WIPHY_RETRY_LONG]) {
                retry_long = nla_get_u8(
                        info->attrs[NL80211_ATTR_WIPHY_RETRY_LONG]);

                changed |= WIPHY_PARAM_RETRY_LONG;
        }

        if (info->attrs[NL80211_ATTR_WIPHY_FRAG_THRESHOLD]) {
                frag_threshold = nla_get_u32(
                        info->attrs[NL80211_ATTR_WIPHY_FRAG_THRESHOLD]);
                if (frag_threshold < 256)
                        return -EINVAL;

                if (frag_threshold != (u32) -1) {
                        /*
                         * Fragments (apart from the last one) are required to
                         * have even length. Make the fragmentation code
                         * simpler by stripping LSB should someone try to use
                         * odd threshold value.
                         */
                        frag_threshold &= ~0x1;
                }
                changed |= WIPHY_PARAM_FRAG_THRESHOLD;
        }

        if (info->attrs[NL80211_ATTR_WIPHY_RTS_THRESHOLD]) {
                rts_threshold = nla_get_u32(
                        info->attrs[NL80211_ATTR_WIPHY_RTS_THRESHOLD]);
                changed |= WIPHY_PARAM_RTS_THRESHOLD;
        }

        if (info->attrs[NL80211_ATTR_WIPHY_COVERAGE_CLASS]) {
                if (info->attrs[NL80211_ATTR_WIPHY_DYN_ACK])
                        return -EINVAL;

                coverage_class = nla_get_u8(
                        info->attrs[NL80211_ATTR_WIPHY_COVERAGE_CLASS]);
                changed |= WIPHY_PARAM_COVERAGE_CLASS;
        }

        if (info->attrs[NL80211_ATTR_WIPHY_DYN_ACK]) {
                if (!(rdev->wiphy.features & NL80211_FEATURE_ACKTO_ESTIMATION))
                        return -EOPNOTSUPP;

                changed |= WIPHY_PARAM_DYN_ACK;
        }

        if (info->attrs[NL80211_ATTR_TXQ_LIMIT]) {
                if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_TXQS))
                        return -EOPNOTSUPP;

                txq_limit = nla_get_u32(
                        info->attrs[NL80211_ATTR_TXQ_LIMIT]);
                changed |= WIPHY_PARAM_TXQ_LIMIT;
        }

        if (info->attrs[NL80211_ATTR_TXQ_MEMORY_LIMIT]) {
                if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_TXQS))
                        return -EOPNOTSUPP;

                txq_memory_limit = nla_get_u32(
                        info->attrs[NL80211_ATTR_TXQ_MEMORY_LIMIT]);
                changed |= WIPHY_PARAM_TXQ_MEMORY_LIMIT;
        }

        if (info->attrs[NL80211_ATTR_TXQ_QUANTUM]) {
                if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_TXQS))
                        return -EOPNOTSUPP;

                txq_quantum = nla_get_u32(
                        info->attrs[NL80211_ATTR_TXQ_QUANTUM]);
                changed |= WIPHY_PARAM_TXQ_QUANTUM;
        }

        if (changed) {
                u8 old_retry_short, old_retry_long;
                u32 old_frag_threshold, old_rts_threshold;
                u8 old_coverage_class, i;
                u32 old_txq_limit, old_txq_memory_limit, old_txq_quantum;
                u32 *old_radio_rts_threshold = NULL;

                if (!rdev->ops->set_wiphy_params)
                        return -EOPNOTSUPP;

                if (rdev->wiphy.n_radio) {
                        old_radio_rts_threshold = kcalloc(rdev->wiphy.n_radio,
                                                          sizeof(u32),
                                                          GFP_KERNEL);
                        if (!old_radio_rts_threshold)
                                return -ENOMEM;
                }

                old_retry_short = rdev->wiphy.retry_short;
                old_retry_long = rdev->wiphy.retry_long;
                old_frag_threshold = rdev->wiphy.frag_threshold;
                old_rts_threshold = rdev->wiphy.rts_threshold;
                if (old_radio_rts_threshold) {
                        for (i = 0 ; i < rdev->wiphy.n_radio; i++)
                                old_radio_rts_threshold[i] =
                                        rdev->wiphy.radio_cfg[i].rts_threshold;
                }
                old_coverage_class = rdev->wiphy.coverage_class;
                old_txq_limit = rdev->wiphy.txq_limit;
                old_txq_memory_limit = rdev->wiphy.txq_memory_limit;
                old_txq_quantum = rdev->wiphy.txq_quantum;

                if (changed & WIPHY_PARAM_RETRY_SHORT)
                        rdev->wiphy.retry_short = retry_short;
                if (changed & WIPHY_PARAM_RETRY_LONG)
                        rdev->wiphy.retry_long = retry_long;
                if (changed & WIPHY_PARAM_FRAG_THRESHOLD)
                        rdev->wiphy.frag_threshold = frag_threshold;
                if ((changed & WIPHY_PARAM_RTS_THRESHOLD) &&
                    old_radio_rts_threshold) {
                        rdev->wiphy.rts_threshold = rts_threshold;
                        for (i = 0 ; i < rdev->wiphy.n_radio; i++)
                                rdev->wiphy.radio_cfg[i].rts_threshold =
                                        rdev->wiphy.rts_threshold;
                }
                if (changed & WIPHY_PARAM_COVERAGE_CLASS)
                        rdev->wiphy.coverage_class = coverage_class;
                if (changed & WIPHY_PARAM_TXQ_LIMIT)
                        rdev->wiphy.txq_limit = txq_limit;
                if (changed & WIPHY_PARAM_TXQ_MEMORY_LIMIT)
                        rdev->wiphy.txq_memory_limit = txq_memory_limit;
                if (changed & WIPHY_PARAM_TXQ_QUANTUM)
                        rdev->wiphy.txq_quantum = txq_quantum;

                result = rdev_set_wiphy_params(rdev, radio_idx, changed);
                if (result) {
                        rdev->wiphy.retry_short = old_retry_short;
                        rdev->wiphy.retry_long = old_retry_long;
                        rdev->wiphy.frag_threshold = old_frag_threshold;
                        rdev->wiphy.rts_threshold = old_rts_threshold;
                        if (old_radio_rts_threshold) {
                                for (i = 0 ; i < rdev->wiphy.n_radio; i++)
                                        rdev->wiphy.radio_cfg[i].rts_threshold =
                                                old_radio_rts_threshold[i];
                        }
                        rdev->wiphy.coverage_class = old_coverage_class;
                        rdev->wiphy.txq_limit = old_txq_limit;
                        rdev->wiphy.txq_memory_limit = old_txq_memory_limit;
                        rdev->wiphy.txq_quantum = old_txq_quantum;
                }

                kfree(old_radio_rts_threshold);
                return result;
        }

        return 0;
}

int nl80211_send_chandef(struct sk_buff *msg, const struct cfg80211_chan_def *chandef)
{
        if (WARN_ON(!cfg80211_chandef_valid(chandef)))
                return -EINVAL;

        if (nla_put_u32(msg, NL80211_ATTR_WIPHY_FREQ,
                        chandef->chan->center_freq))
                return -ENOBUFS;
        if (nla_put_u32(msg, NL80211_ATTR_WIPHY_FREQ_OFFSET,
                        chandef->chan->freq_offset))
                return -ENOBUFS;
        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_20_NOHT:
        case NL80211_CHAN_WIDTH_20:
        case NL80211_CHAN_WIDTH_40:
                if (nla_put_u32(msg, NL80211_ATTR_WIPHY_CHANNEL_TYPE,
                                cfg80211_get_chandef_type(chandef)))
                        return -ENOBUFS;
                break;
        default:
                break;
        }
        if (nla_put_u32(msg, NL80211_ATTR_CHANNEL_WIDTH, chandef->width))
                return -ENOBUFS;
        if (nla_put_u32(msg, NL80211_ATTR_CENTER_FREQ1, chandef->center_freq1))
                return -ENOBUFS;
        if (chandef->freq1_offset &&
            nla_put_u32(msg, NL80211_ATTR_CENTER_FREQ1_OFFSET,
                        chandef->freq1_offset))
                return -ENOBUFS;
        if (chandef->center_freq2 &&
            nla_put_u32(msg, NL80211_ATTR_CENTER_FREQ2, chandef->center_freq2))
                return -ENOBUFS;
        if (chandef->punctured &&
            nla_put_u32(msg, NL80211_ATTR_PUNCT_BITMAP, chandef->punctured))
                return -ENOBUFS;
        if (chandef->s1g_primary_2mhz &&
            nla_put_flag(msg, NL80211_ATTR_S1G_PRIMARY_2MHZ))
                return -ENOBUFS;

        if (chandef->npca_chan &&
            nla_put_u32(msg, NL80211_ATTR_NPCA_PRIMARY_FREQ,
                        chandef->npca_chan->center_freq))
                return -ENOBUFS;
        if (chandef->npca_punctured &&
            nla_put_u32(msg, NL80211_ATTR_NPCA_PUNCT_BITMAP,
                        chandef->npca_punctured))
                return -ENOBUFS;

        return 0;
}
EXPORT_SYMBOL(nl80211_send_chandef);

static int nl80211_send_iface(struct sk_buff *msg, u32 portid, u32 seq, int flags,
                              struct cfg80211_registered_device *rdev,
                              struct wireless_dev *wdev,
                              enum nl80211_commands cmd)
{
        struct net_device *dev = wdev->netdev;
        void *hdr;

        lockdep_assert_wiphy(&rdev->wiphy);

        WARN_ON(cmd != NL80211_CMD_NEW_INTERFACE &&
                cmd != NL80211_CMD_DEL_INTERFACE &&
                cmd != NL80211_CMD_SET_INTERFACE);

        hdr = nl80211hdr_put(msg, portid, seq, flags, cmd);
        if (!hdr)
                return -1;

        if (dev &&
            (nla_put_u32(msg, NL80211_ATTR_IFINDEX, dev->ifindex) ||
             nla_put_string(msg, NL80211_ATTR_IFNAME, dev->name)))
                goto nla_put_failure;

        if (nla_put_u32(msg, NL80211_ATTR_WIPHY, rdev->wiphy_idx) ||
            nla_put_u32(msg, NL80211_ATTR_IFTYPE, wdev->iftype) ||
            nla_put_u64_64bit(msg, NL80211_ATTR_WDEV, wdev_id(wdev),
                              NL80211_ATTR_PAD) ||
            nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, wdev_address(wdev)) ||
            nla_put_u32(msg, NL80211_ATTR_GENERATION,
                        rdev->devlist_generation ^
                        (cfg80211_rdev_list_generation << 2)) ||
            nla_put_u8(msg, NL80211_ATTR_4ADDR, wdev->use_4addr) ||
            nla_put_u32(msg, NL80211_ATTR_VIF_RADIO_MASK, wdev->radio_mask))
                goto nla_put_failure;

        if (rdev->ops->get_channel && !wdev->valid_links) {
                struct cfg80211_chan_def chandef = {};
                int ret;

                ret = rdev_get_channel(rdev, wdev, 0, &chandef);
                if (ret == 0 && nl80211_send_chandef(msg, &chandef))
                        goto nla_put_failure;
        }

        if (rdev->ops->get_tx_power && !wdev->valid_links) {
                int dbm, ret;

                ret = rdev_get_tx_power(rdev, wdev, -1, 0, &dbm);
                if (ret == 0 &&
                    nla_put_u32(msg, NL80211_ATTR_WIPHY_TX_POWER_LEVEL,
                                DBM_TO_MBM(dbm)))
                        goto nla_put_failure;
        }

        switch (wdev->iftype) {
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_P2P_GO:
                if (wdev->u.ap.ssid_len &&
                    nla_put(msg, NL80211_ATTR_SSID, wdev->u.ap.ssid_len,
                            wdev->u.ap.ssid))
                        goto nla_put_failure;
                break;
        case NL80211_IFTYPE_STATION:
        case NL80211_IFTYPE_P2P_CLIENT:
                if (wdev->u.client.ssid_len &&
                    nla_put(msg, NL80211_ATTR_SSID, wdev->u.client.ssid_len,
                            wdev->u.client.ssid))
                        goto nla_put_failure;
                break;
        case NL80211_IFTYPE_ADHOC:
                if (wdev->u.ibss.ssid_len &&
                    nla_put(msg, NL80211_ATTR_SSID, wdev->u.ibss.ssid_len,
                            wdev->u.ibss.ssid))
                        goto nla_put_failure;
                break;
        default:
                /* nothing */
                break;
        }

        if (rdev->ops->get_txq_stats) {
                struct cfg80211_txq_stats txqstats = {};
                int ret = rdev_get_txq_stats(rdev, wdev, &txqstats);

                if (ret == 0 &&
                    !nl80211_put_txq_stats(msg, &txqstats,
                                           NL80211_ATTR_TXQ_STATS))
                        goto nla_put_failure;
        }

        if (wdev->valid_links) {
                unsigned int link_id;
                struct nlattr *links = nla_nest_start(msg,
                                                      NL80211_ATTR_MLO_LINKS);

                if (!links)
                        goto nla_put_failure;

                for_each_valid_link(wdev, link_id) {
                        struct nlattr *link = nla_nest_start(msg, link_id + 1);
                        struct cfg80211_chan_def chandef = {};
                        int ret;

                        if (!link)
                                goto nla_put_failure;

                        if (nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_id))
                                goto nla_put_failure;
                        if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN,
                                    wdev->links[link_id].addr))
                                goto nla_put_failure;

                        ret = rdev_get_channel(rdev, wdev, link_id, &chandef);
                        if (ret == 0 && nl80211_send_chandef(msg, &chandef))
                                goto nla_put_failure;

                        if (rdev->ops->get_tx_power) {
                                int dbm, ret;

                                ret = rdev_get_tx_power(rdev, wdev, -1, link_id, &dbm);
                                if (ret == 0 &&
                                    nla_put_u32(msg, NL80211_ATTR_WIPHY_TX_POWER_LEVEL,
                                                DBM_TO_MBM(dbm)))
                                        goto nla_put_failure;
                        }
                        nla_nest_end(msg, link);
                }

                nla_nest_end(msg, links);
        }

        genlmsg_end(msg, hdr);
        return 0;

 nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

static int nl80211_dump_interface(struct sk_buff *skb, struct netlink_callback *cb)
{
        int wp_idx = 0;
        int if_idx = 0;
        int wp_start = cb->args[0];
        int if_start = cb->args[1];
        int filter_wiphy = -1;
        struct cfg80211_registered_device *rdev;
        struct wireless_dev *wdev;
        int ret;

        rtnl_lock();
        if (!cb->args[2]) {
                struct nl80211_dump_wiphy_state state = {
                        .filter_wiphy = -1,
                };

                ret = nl80211_dump_wiphy_parse(skb, cb, &state);
                if (ret)
                        goto out_unlock;

                filter_wiphy = state.filter_wiphy;

                /*
                 * if filtering, set cb->args[2] to +1 since 0 is the default
                 * value needed to determine that parsing is necessary.
                 */
                if (filter_wiphy >= 0)
                        cb->args[2] = filter_wiphy + 1;
                else
                        cb->args[2] = -1;
        } else if (cb->args[2] > 0) {
                filter_wiphy = cb->args[2] - 1;
        }

        for_each_rdev(rdev) {
                if (!net_eq(wiphy_net(&rdev->wiphy), sock_net(skb->sk)))
                        continue;
                if (wp_idx < wp_start) {
                        wp_idx++;
                        continue;
                }

                if (filter_wiphy >= 0 && filter_wiphy != rdev->wiphy_idx)
                        continue;

                if_idx = 0;

                guard(wiphy)(&rdev->wiphy);

                list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                        if (if_idx < if_start) {
                                if_idx++;
                                continue;
                        }

                        if (nl80211_send_iface(skb, NETLINK_CB(cb->skb).portid,
                                               cb->nlh->nlmsg_seq, NLM_F_MULTI,
                                               rdev, wdev,
                                               NL80211_CMD_NEW_INTERFACE) < 0)
                                goto out;

                        if_idx++;
                }

                if_start = 0;
                wp_idx++;
        }
 out:
        cb->args[0] = wp_idx;
        cb->args[1] = if_idx;

        ret = skb->len;
 out_unlock:
        rtnl_unlock();

        return ret;
}

static int nl80211_get_interface(struct sk_buff *skb, struct genl_info *info)
{
        struct sk_buff *msg;
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct wireless_dev *wdev = info->user_ptr[1];

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        if (nl80211_send_iface(msg, info->snd_portid, info->snd_seq, 0,
                               rdev, wdev, NL80211_CMD_NEW_INTERFACE) < 0) {
                nlmsg_free(msg);
                return -ENOBUFS;
        }

        return genlmsg_reply(msg, info);
}

static const struct nla_policy mntr_flags_policy[NL80211_MNTR_FLAG_MAX + 1] = {
        [NL80211_MNTR_FLAG_FCSFAIL] = { .type = NLA_FLAG },
        [NL80211_MNTR_FLAG_PLCPFAIL] = { .type = NLA_FLAG },
        [NL80211_MNTR_FLAG_CONTROL] = { .type = NLA_FLAG },
        [NL80211_MNTR_FLAG_OTHER_BSS] = { .type = NLA_FLAG },
        [NL80211_MNTR_FLAG_COOK_FRAMES] = { .type = NLA_FLAG },
        [NL80211_MNTR_FLAG_ACTIVE] = { .type = NLA_FLAG },
        [NL80211_MNTR_FLAG_SKIP_TX] = { .type = NLA_FLAG },
};

static int parse_monitor_flags(struct nlattr *nla, u32 *mntrflags)
{
        struct nlattr *flags[NL80211_MNTR_FLAG_MAX + 1];
        int flag;

        *mntrflags = 0;

        if (!nla)
                return -EINVAL;

        if (nla_parse_nested_deprecated(flags, NL80211_MNTR_FLAG_MAX, nla, mntr_flags_policy, NULL))
                return -EINVAL;

        for (flag = 1; flag <= NL80211_MNTR_FLAG_MAX; flag++)
                if (flags[flag])
                        *mntrflags |= (1<<flag);

        /* cooked monitor mode is incompatible with other modes */
        if (*mntrflags & MONITOR_FLAG_COOK_FRAMES &&
            *mntrflags != MONITOR_FLAG_COOK_FRAMES)
                return -EOPNOTSUPP;

        *mntrflags |= MONITOR_FLAG_CHANGED;

        return 0;
}

static int nl80211_parse_mon_options(struct cfg80211_registered_device *rdev,
                                     enum nl80211_iftype type,
                                     struct genl_info *info,
                                     struct vif_params *params)
{
        bool change = false;
        int err;

        if (info->attrs[NL80211_ATTR_MNTR_FLAGS]) {
                if (type != NL80211_IFTYPE_MONITOR)
                        return -EINVAL;

                err = parse_monitor_flags(info->attrs[NL80211_ATTR_MNTR_FLAGS],
                                          &params->flags);
                if (err)
                        return err;

                change = true;
        }

        /* MONITOR_FLAG_COOK_FRAMES is deprecated, refuse cooperation */
        if (params->flags & MONITOR_FLAG_COOK_FRAMES)
                return -EOPNOTSUPP;

        if (params->flags & MONITOR_FLAG_ACTIVE &&
            !(rdev->wiphy.features & NL80211_FEATURE_ACTIVE_MONITOR))
                return -EOPNOTSUPP;

        if (info->attrs[NL80211_ATTR_MU_MIMO_GROUP_DATA]) {
                const u8 *mumimo_groups;
                u32 cap_flag = NL80211_EXT_FEATURE_MU_MIMO_AIR_SNIFFER;

                if (type != NL80211_IFTYPE_MONITOR)
                        return -EINVAL;

                if (!wiphy_ext_feature_isset(&rdev->wiphy, cap_flag))
                        return -EOPNOTSUPP;

                mumimo_groups =
                        nla_data(info->attrs[NL80211_ATTR_MU_MIMO_GROUP_DATA]);

                /* bits 0 and 63 are reserved and must be zero */
                if ((mumimo_groups[0] & BIT(0)) ||
                    (mumimo_groups[VHT_MUMIMO_GROUPS_DATA_LEN - 1] & BIT(7)))
                        return -EINVAL;

                params->vht_mumimo_groups = mumimo_groups;
                change = true;
        }

        if (info->attrs[NL80211_ATTR_MU_MIMO_FOLLOW_MAC_ADDR]) {
                u32 cap_flag = NL80211_EXT_FEATURE_MU_MIMO_AIR_SNIFFER;

                if (type != NL80211_IFTYPE_MONITOR)
                        return -EINVAL;

                if (!wiphy_ext_feature_isset(&rdev->wiphy, cap_flag))
                        return -EOPNOTSUPP;

                params->vht_mumimo_follow_addr =
                        nla_data(info->attrs[NL80211_ATTR_MU_MIMO_FOLLOW_MAC_ADDR]);
                change = true;
        }

        return change ? 1 : 0;
}

static int nl80211_valid_4addr(struct cfg80211_registered_device *rdev,
                               struct net_device *netdev, u8 use_4addr,
                               enum nl80211_iftype iftype)
{
        if (!use_4addr) {
                if (netdev && netif_is_bridge_port(netdev))
                        return -EBUSY;
                return 0;
        }

        switch (iftype) {
        case NL80211_IFTYPE_AP_VLAN:
                if (rdev->wiphy.flags & WIPHY_FLAG_4ADDR_AP)
                        return 0;
                break;
        case NL80211_IFTYPE_STATION:
                if (rdev->wiphy.flags & WIPHY_FLAG_4ADDR_STATION)
                        return 0;
                break;
        default:
                break;
        }

        return -EOPNOTSUPP;
}

static int nl80211_parse_vif_radio_mask(struct genl_info *info,
                                        u32 *radio_mask)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct nlattr *attr = info->attrs[NL80211_ATTR_VIF_RADIO_MASK];
        u32 mask, allowed;

        if (!attr) {
                *radio_mask = 0;
                return 0;
        }

        allowed = BIT(rdev->wiphy.n_radio) - 1;
        mask = nla_get_u32(attr);
        if (mask & ~allowed)
                return -EINVAL;
        if (!mask)
                mask = allowed;
        *radio_mask = mask;

        return 1;
}

static int nl80211_set_interface(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct vif_params params;
        int err;
        enum nl80211_iftype otype, ntype;
        struct net_device *dev = info->user_ptr[1];
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        u32 radio_mask = 0;
        bool change = false;

        memset(&params, 0, sizeof(params));

        otype = ntype = dev->ieee80211_ptr->iftype;

        if (info->attrs[NL80211_ATTR_IFTYPE]) {
                ntype = nla_get_u32(info->attrs[NL80211_ATTR_IFTYPE]);
                if (otype != ntype)
                        change = true;
        }

        if (info->attrs[NL80211_ATTR_MESH_ID]) {
                if (ntype != NL80211_IFTYPE_MESH_POINT)
                        return -EINVAL;
                if (otype != NL80211_IFTYPE_MESH_POINT)
                        return -EINVAL;
                if (netif_running(dev))
                        return -EBUSY;

                wdev->u.mesh.id_up_len =
                        nla_len(info->attrs[NL80211_ATTR_MESH_ID]);
                memcpy(wdev->u.mesh.id,
                       nla_data(info->attrs[NL80211_ATTR_MESH_ID]),
                       wdev->u.mesh.id_up_len);
        }

        if (info->attrs[NL80211_ATTR_4ADDR]) {
                params.use_4addr = !!nla_get_u8(info->attrs[NL80211_ATTR_4ADDR]);
                change = true;
                err = nl80211_valid_4addr(rdev, dev, params.use_4addr, ntype);
                if (err)
                        return err;
        } else {
                params.use_4addr = -1;
        }

        err = nl80211_parse_mon_options(rdev, ntype, info, &params);
        if (err < 0)
                return err;
        if (err > 0)
                change = true;

        err = nl80211_parse_vif_radio_mask(info, &radio_mask);
        if (err < 0)
                return err;
        if (err && netif_running(dev))
                return -EBUSY;

        if (change)
                err = cfg80211_change_iface(rdev, dev, ntype, &params);
        else
                err = 0;

        if (!err && params.use_4addr != -1)
                dev->ieee80211_ptr->use_4addr = params.use_4addr;

        if (radio_mask)
                wdev->radio_mask = radio_mask;

        if (change && !err)
                nl80211_notify_iface(rdev, wdev, NL80211_CMD_SET_INTERFACE);

        return err;
}

static int _nl80211_new_interface(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct vif_params params;
        struct wireless_dev *wdev;
        struct sk_buff *msg;
        u32 radio_mask;
        int err;
        enum nl80211_iftype type = NL80211_IFTYPE_UNSPECIFIED;

        memset(&params, 0, sizeof(params));

        if (!info->attrs[NL80211_ATTR_IFNAME])
                return -EINVAL;

        if (info->attrs[NL80211_ATTR_IFTYPE])
                type = nla_get_u32(info->attrs[NL80211_ATTR_IFTYPE]);

        if (!rdev->ops->add_virtual_intf)
                return -EOPNOTSUPP;

        if ((type == NL80211_IFTYPE_P2P_DEVICE || type == NL80211_IFTYPE_NAN ||
             type == NL80211_IFTYPE_PD ||
             rdev->wiphy.features & NL80211_FEATURE_MAC_ON_CREATE) &&
            info->attrs[NL80211_ATTR_MAC]) {
                nla_memcpy(params.macaddr, info->attrs[NL80211_ATTR_MAC],
                           ETH_ALEN);
                if (!is_valid_ether_addr(params.macaddr))
                        return -EADDRNOTAVAIL;
        }

        if (info->attrs[NL80211_ATTR_4ADDR]) {
                params.use_4addr = !!nla_get_u8(info->attrs[NL80211_ATTR_4ADDR]);
                err = nl80211_valid_4addr(rdev, NULL, params.use_4addr, type);
                if (err)
                        return err;
        }

        if (!cfg80211_iftype_allowed(&rdev->wiphy, type, params.use_4addr, 0))
                return -EOPNOTSUPP;

        err = nl80211_parse_mon_options(rdev, type, info, &params);
        if (err < 0)
                return err;

        err = nl80211_parse_vif_radio_mask(info, &radio_mask);
        if (err < 0)
                return err;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        wdev = rdev_add_virtual_intf(rdev,
                                nla_data(info->attrs[NL80211_ATTR_IFNAME]),
                                NET_NAME_USER, type, &params);
        if (WARN_ON(!wdev)) {
                nlmsg_free(msg);
                return -EPROTO;
        } else if (IS_ERR(wdev)) {
                nlmsg_free(msg);
                return PTR_ERR(wdev);
        }

        if (info->attrs[NL80211_ATTR_SOCKET_OWNER])
                wdev->owner_nlportid = info->snd_portid;

        switch (type) {
        case NL80211_IFTYPE_MESH_POINT:
                if (!info->attrs[NL80211_ATTR_MESH_ID])
                        break;
                wdev->u.mesh.id_up_len =
                        nla_len(info->attrs[NL80211_ATTR_MESH_ID]);
                memcpy(wdev->u.mesh.id,
                       nla_data(info->attrs[NL80211_ATTR_MESH_ID]),
                       wdev->u.mesh.id_up_len);
                break;
        case NL80211_IFTYPE_NAN:
        case NL80211_IFTYPE_P2P_DEVICE:
        case NL80211_IFTYPE_PD:
                /*
                 * P2P Device, NAN and PD do not have a netdev, so don't go
                 * through the netdev notifier and must be added here
                 */
                cfg80211_init_wdev(wdev);
                cfg80211_register_wdev(rdev, wdev);
                break;
        default:
                break;
        }

        if (radio_mask)
                wdev->radio_mask = radio_mask;

        if (nl80211_send_iface(msg, info->snd_portid, info->snd_seq, 0,
                               rdev, wdev, NL80211_CMD_NEW_INTERFACE) < 0) {
                nlmsg_free(msg);
                return -ENOBUFS;
        }

        return genlmsg_reply(msg, info);
}

static int nl80211_new_interface(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];

        /* to avoid failing a new interface creation due to pending removal */
        cfg80211_destroy_ifaces(rdev);

        guard(wiphy)(&rdev->wiphy);

        return _nl80211_new_interface(skb, info);
}

static int nl80211_del_interface(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct wireless_dev *wdev = info->user_ptr[1];

        if (!rdev->ops->del_virtual_intf)
                return -EOPNOTSUPP;

        /*
         * We hold RTNL, so this is safe, without RTNL opencount cannot
         * reach 0, and thus the rdev cannot be deleted.
         *
         * We need to do it for the dev_close(), since that will call
         * the netdev notifiers, and we need to acquire the mutex there
         * but don't know if we get there from here or from some other
         * place (e.g. "ip link set ... down").
         */
        mutex_unlock(&rdev->wiphy.mtx);

        /*
         * If we remove a wireless device without a netdev then clear
         * user_ptr[1] so that nl80211_post_doit won't dereference it
         * to check if it needs to do dev_put(). Otherwise it crashes
         * since the wdev has been freed, unlike with a netdev where
         * we need the dev_put() for the netdev to really be freed.
         */
        if (!wdev->netdev)
                info->user_ptr[1] = NULL;
        else
                dev_close(wdev->netdev);

        cfg80211_close_dependents(rdev, wdev);

        mutex_lock(&rdev->wiphy.mtx);

        return cfg80211_remove_virtual_intf(rdev, wdev);
}

static int nl80211_set_noack_map(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct net_device *dev = info->user_ptr[1];
        u16 noack_map;

        if (!info->attrs[NL80211_ATTR_NOACK_MAP])
                return -EINVAL;

        if (!rdev->ops->set_noack_map)
                return -EOPNOTSUPP;

        noack_map = nla_get_u16(info->attrs[NL80211_ATTR_NOACK_MAP]);

        return rdev_set_noack_map(rdev, dev, noack_map);
}

static int nl80211_validate_key_link_id(struct genl_info *info,
                                        struct wireless_dev *wdev,
                                        int link_id, bool pairwise)
{
        if (pairwise) {
                if (link_id != -1) {
                        GENL_SET_ERR_MSG(info,
                                         "link ID not allowed for pairwise key");
                        return -EINVAL;
                }

                return 0;
        }

        if (wdev->valid_links) {
                if (link_id == -1) {
                        GENL_SET_ERR_MSG(info,
                                         "link ID must be set for MLO group key");
                        return -EINVAL;
                }
                if (!(wdev->valid_links & BIT(link_id))) {
                        GENL_SET_ERR_MSG(info, "invalid link ID for MLO group key");
                        return -EINVAL;
                }
        } else if (link_id != -1) {
                GENL_SET_ERR_MSG(info, "link ID not allowed for non-MLO group key");
                return -EINVAL;
        }

        return 0;
}

struct get_key_cookie {
        struct sk_buff *msg;
        int error;
        int idx;
};

static void get_key_callback(void *c, struct key_params *params)
{
        struct nlattr *key;
        struct get_key_cookie *cookie = c;

        if ((params->seq &&
             nla_put(cookie->msg, NL80211_ATTR_KEY_SEQ,
                     params->seq_len, params->seq)) ||
            (params->cipher &&
             nla_put_u32(cookie->msg, NL80211_ATTR_KEY_CIPHER,
                         params->cipher)))
                goto nla_put_failure;

        key = nla_nest_start_noflag(cookie->msg, NL80211_ATTR_KEY);
        if (!key)
                goto nla_put_failure;

        if ((params->seq &&
             nla_put(cookie->msg, NL80211_KEY_SEQ,
                     params->seq_len, params->seq)) ||
            (params->cipher &&
             nla_put_u32(cookie->msg, NL80211_KEY_CIPHER,
                         params->cipher)))
                goto nla_put_failure;

        if (nla_put_u8(cookie->msg, NL80211_KEY_IDX, cookie->idx))
                goto nla_put_failure;

        nla_nest_end(cookie->msg, key);

        return;
 nla_put_failure:
        cookie->error = 1;
}

static int nl80211_get_key(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        int err;
        struct wireless_dev *wdev = info->user_ptr[1];
        u8 key_idx = 0;
        const u8 *mac_addr = NULL;
        bool pairwise;
        struct get_key_cookie cookie = {
                .error = 0,
        };
        void *hdr;
        struct sk_buff *msg;
        bool bigtk_support = false;
        int link_id = nl80211_link_id_or_invalid(info->attrs);

        if (wiphy_ext_feature_isset(&rdev->wiphy,
                                    NL80211_EXT_FEATURE_BEACON_PROTECTION))
                bigtk_support = true;

        if ((wdev->iftype == NL80211_IFTYPE_STATION ||
             wdev->iftype == NL80211_IFTYPE_P2P_CLIENT) &&
            wiphy_ext_feature_isset(&rdev->wiphy,
                                    NL80211_EXT_FEATURE_BEACON_PROTECTION_CLIENT))
                bigtk_support = true;

        if (info->attrs[NL80211_ATTR_KEY_IDX]) {
                key_idx = nla_get_u8(info->attrs[NL80211_ATTR_KEY_IDX]);

                if (key_idx >= 6 && key_idx <= 7 && !bigtk_support) {
                        GENL_SET_ERR_MSG(info, "BIGTK not supported");
                        return -EINVAL;
                }
        }

        if (info->attrs[NL80211_ATTR_MAC])
                mac_addr = nla_data(info->attrs[NL80211_ATTR_MAC]);

        pairwise = !!mac_addr;
        if (info->attrs[NL80211_ATTR_KEY_TYPE]) {
                u32 kt = nla_get_u32(info->attrs[NL80211_ATTR_KEY_TYPE]);

                if (kt != NL80211_KEYTYPE_GROUP &&
                    kt != NL80211_KEYTYPE_PAIRWISE)
                        return -EINVAL;
                pairwise = kt == NL80211_KEYTYPE_PAIRWISE;
        }

        if (!rdev->ops->get_key)
                return -EOPNOTSUPP;

        if (!cfg80211_valid_key_idx(wdev, key_idx, pairwise, mac_addr))
                return -ENOENT;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        hdr = nl80211hdr_put(msg, info->snd_portid, info->snd_seq, 0,
                             NL80211_CMD_NEW_KEY);
        if (!hdr)
                goto nla_put_failure;

        cookie.msg = msg;
        cookie.idx = key_idx;

        if ((wdev->netdev &&
             nla_put_u32(msg, NL80211_ATTR_IFINDEX, wdev->netdev->ifindex)) ||
            nla_put_u64_64bit(msg, NL80211_ATTR_WDEV, wdev_id(wdev),
                              NL80211_ATTR_PAD) ||
            nla_put_u8(msg, NL80211_ATTR_KEY_IDX, key_idx))
                goto nla_put_failure;
        if (mac_addr &&
            nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, mac_addr))
                goto nla_put_failure;

        err = nl80211_validate_key_link_id(info, wdev, link_id, pairwise);
        if (err)
                goto free_msg;

        err = rdev_get_key(rdev, wdev, link_id, key_idx, pairwise, mac_addr,
                           &cookie, get_key_callback);

        if (err)
                goto free_msg;

        if (cookie.error)
                goto nla_put_failure;

        genlmsg_end(msg, hdr);
        return genlmsg_reply(msg, info);

 nla_put_failure:
        err = -ENOBUFS;
 free_msg:
        nlmsg_free(msg);
        return err;
}

static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct key_parse key;
        int err;
        struct wireless_dev *wdev = info->user_ptr[1];
        int link_id = nl80211_link_id_or_invalid(info->attrs);

        err = nl80211_parse_key(info, &key);
        if (err)
                return err;

        if (key.idx < 0)
                return -EINVAL;

        /* Only support setting default key and
         * Extended Key ID action NL80211_KEY_SET_TX.
         */
        if (!key.def && !key.defmgmt && !key.defbeacon &&
            !(key.p.mode == NL80211_KEY_SET_TX))
                return -EINVAL;

        if (key.def) {
                if (!rdev->ops->set_default_key)
                        return -EOPNOTSUPP;

                if (!wdev->netdev)
                        return -EINVAL;

                err = nl80211_key_allowed(wdev);
                if (err)
                        return err;

                err = nl80211_validate_key_link_id(info, wdev, link_id, false);
                if (err)
                        return err;

                err = rdev_set_default_key(rdev, wdev->netdev, link_id, key.idx,
                                           key.def_uni, key.def_multi);

                if (err)
                        return err;

#ifdef CONFIG_CFG80211_WEXT
                wdev->wext.default_key = key.idx;
#endif
                return 0;
        } else if (key.defmgmt) {
                if (key.def_uni || !key.def_multi)
                        return -EINVAL;

                if (!rdev->ops->set_default_mgmt_key)
                        return -EOPNOTSUPP;

                err = nl80211_key_allowed(wdev);
                if (err)
                        return err;

                err = nl80211_validate_key_link_id(info, wdev, link_id, false);
                if (err)
                        return err;

                err = rdev_set_default_mgmt_key(rdev, wdev, link_id, key.idx);
                if (err)
                        return err;

#ifdef CONFIG_CFG80211_WEXT
                wdev->wext.default_mgmt_key = key.idx;
#endif
                return 0;
        } else if (key.defbeacon) {
                if (key.def_uni || !key.def_multi)
                        return -EINVAL;

                if (!rdev->ops->set_default_beacon_key)
                        return -EOPNOTSUPP;

                err = nl80211_key_allowed(wdev);
                if (err)
                        return err;

                err = nl80211_validate_key_link_id(info, wdev, link_id, false);
                if (err)
                        return err;

                return rdev_set_default_beacon_key(rdev, wdev, link_id,
                                                   key.idx);
        } else if (key.p.mode == NL80211_KEY_SET_TX &&
                   wiphy_ext_feature_isset(&rdev->wiphy,
                                           NL80211_EXT_FEATURE_EXT_KEY_ID)) {
                u8 *mac_addr = NULL;

                if (info->attrs[NL80211_ATTR_MAC])
                        mac_addr = nla_data(info->attrs[NL80211_ATTR_MAC]);

                if (!mac_addr || key.idx < 0 || key.idx > 1)
                        return -EINVAL;

                err = nl80211_validate_key_link_id(info, wdev, link_id, true);
                if (err)
                        return err;

                return rdev_add_key(rdev, wdev, link_id, key.idx,
                                    NL80211_KEYTYPE_PAIRWISE,
                                    mac_addr, &key.p);
        }

        return -EINVAL;
}

static int nl80211_new_key(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        int err;
        struct wireless_dev *wdev = info->user_ptr[1];
        struct key_parse key;
        const u8 *mac_addr = NULL;
        int link_id = nl80211_link_id_or_invalid(info->attrs);

        err = nl80211_parse_key(info, &key);
        if (err)
                return err;

        if (!key.p.key) {
                GENL_SET_ERR_MSG(info, "no key");
                return -EINVAL;
        }

        if (info->attrs[NL80211_ATTR_MAC])
                mac_addr = nla_data(info->attrs[NL80211_ATTR_MAC]);

        if (key.type == -1) {
                if (mac_addr)
                        key.type = NL80211_KEYTYPE_PAIRWISE;
                else
                        key.type = NL80211_KEYTYPE_GROUP;
        }

        /* for now */
        if (key.type != NL80211_KEYTYPE_PAIRWISE &&
            key.type != NL80211_KEYTYPE_GROUP) {
                GENL_SET_ERR_MSG(info, "key type not pairwise or group");
                return -EINVAL;
        }

        if (key.type == NL80211_KEYTYPE_GROUP &&
            info->attrs[NL80211_ATTR_VLAN_ID])
                key.p.vlan_id = nla_get_u16(info->attrs[NL80211_ATTR_VLAN_ID]);

        if (!rdev->ops->add_key)
                return -EOPNOTSUPP;

        if (cfg80211_validate_key_settings(rdev, wdev, &key.p, key.idx,
                                           key.type == NL80211_KEYTYPE_PAIRWISE,
                                           mac_addr)) {
                GENL_SET_ERR_MSG(info, "key setting validation failed");
                return -EINVAL;
        }

        err = nl80211_key_allowed(wdev);
        if (err)
                GENL_SET_ERR_MSG(info, "key not allowed");

        if (!err)
                err = nl80211_validate_key_link_id(info, wdev, link_id,
                                key.type == NL80211_KEYTYPE_PAIRWISE);

        if (!err) {
                err = rdev_add_key(rdev, wdev, link_id, key.idx,
                                   key.type == NL80211_KEYTYPE_PAIRWISE,
                                    mac_addr, &key.p);
                if (err)
                        GENL_SET_ERR_MSG(info, "key addition failed");
        }

        return err;
}

static int nl80211_del_key(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        int err;
        struct wireless_dev *wdev = info->user_ptr[1];
        u8 *mac_addr = NULL;
        struct key_parse key;
        int link_id = nl80211_link_id_or_invalid(info->attrs);

        err = nl80211_parse_key(info, &key);
        if (err)
                return err;

        if (info->attrs[NL80211_ATTR_MAC])
                mac_addr = nla_data(info->attrs[NL80211_ATTR_MAC]);

        if (key.type == -1) {
                if (mac_addr)
                        key.type = NL80211_KEYTYPE_PAIRWISE;
                else
                        key.type = NL80211_KEYTYPE_GROUP;
        }

        /* for now */
        if (key.type != NL80211_KEYTYPE_PAIRWISE &&
            key.type != NL80211_KEYTYPE_GROUP)
                return -EINVAL;

        if (!cfg80211_valid_key_idx(wdev, key.idx,
                                    key.type == NL80211_KEYTYPE_PAIRWISE,
                                    mac_addr))
                return -EINVAL;

        if (!rdev->ops->del_key)
                return -EOPNOTSUPP;

        err = nl80211_key_allowed(wdev);

        if (!err)
                err = nl80211_validate_key_link_id(info, wdev, link_id,
                                key.type == NL80211_KEYTYPE_PAIRWISE);

        if (!err)
                err = rdev_del_key(rdev, wdev, link_id, key.idx,
                                   key.type == NL80211_KEYTYPE_PAIRWISE,
                                   mac_addr);

#ifdef CONFIG_CFG80211_WEXT
        if (!err) {
                if (key.idx == wdev->wext.default_key)
                        wdev->wext.default_key = -1;
                else if (key.idx == wdev->wext.default_mgmt_key)
                        wdev->wext.default_mgmt_key = -1;
        }
#endif

        return err;
}

/* This function returns an error or the number of nested attributes */
static int validate_acl_mac_addrs(struct nlattr *nl_attr)
{
        struct nlattr *attr;
        int n_entries = 0, tmp;

        nla_for_each_nested(attr, nl_attr, tmp) {
                if (nla_len(attr) != ETH_ALEN)
                        return -EINVAL;

                n_entries++;
        }

        return n_entries;
}

/*
 * This function parses ACL information and allocates memory for ACL data.
 * On successful return, the calling function is responsible to free the
 * ACL buffer returned by this function.
 */
static struct cfg80211_acl_data *parse_acl_data(struct wiphy *wiphy,
                                                struct genl_info *info)
{
        enum nl80211_acl_policy acl_policy;
        struct nlattr *attr;
        struct cfg80211_acl_data *acl;
        int i = 0, n_entries, tmp;

        if (!wiphy->max_acl_mac_addrs)
                return ERR_PTR(-EOPNOTSUPP);

        if (!info->attrs[NL80211_ATTR_ACL_POLICY])
                return ERR_PTR(-EINVAL);

        acl_policy = nla_get_u32(info->attrs[NL80211_ATTR_ACL_POLICY]);
        if (acl_policy != NL80211_ACL_POLICY_ACCEPT_UNLESS_LISTED &&
            acl_policy != NL80211_ACL_POLICY_DENY_UNLESS_LISTED)
                return ERR_PTR(-EINVAL);

        if (!info->attrs[NL80211_ATTR_MAC_ADDRS])
                return ERR_PTR(-EINVAL);

        n_entries = validate_acl_mac_addrs(info->attrs[NL80211_ATTR_MAC_ADDRS]);
        if (n_entries < 0)
                return ERR_PTR(n_entries);

        if (n_entries > wiphy->max_acl_mac_addrs)
                return ERR_PTR(-EOPNOTSUPP);

        acl = kzalloc_flex(*acl, mac_addrs, n_entries);
        if (!acl)
                return ERR_PTR(-ENOMEM);
        acl->n_acl_entries = n_entries;

        nla_for_each_nested(attr, info->attrs[NL80211_ATTR_MAC_ADDRS], tmp) {
                memcpy(acl->mac_addrs[i].addr, nla_data(attr), ETH_ALEN);
                i++;
        }
        acl->acl_policy = acl_policy;

        return acl;
}

static int nl80211_set_mac_acl(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct net_device *dev = info->user_ptr[1];
        struct cfg80211_acl_data *acl;
        int err;

        if (dev->ieee80211_ptr->iftype != NL80211_IFTYPE_AP &&
            dev->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_GO)
                return -EOPNOTSUPP;

        if (!dev->ieee80211_ptr->links[0].ap.beacon_interval)
                return -EINVAL;

        acl = parse_acl_data(&rdev->wiphy, info);
        if (IS_ERR(acl))
                return PTR_ERR(acl);

        err = rdev_set_mac_acl(rdev, dev, acl);

        kfree(acl);

        return err;
}

static u32 rateset_to_mask(struct ieee80211_supported_band *sband,
                           u8 *rates, u8 rates_len)
{
        u8 i;
        u32 mask = 0;

        for (i = 0; i < rates_len; i++) {
                int rate = (rates[i] & 0x7f) * 5;
                int ridx;

                for (ridx = 0; ridx < sband->n_bitrates; ridx++) {
                        struct ieee80211_rate *srate =
                                &sband->bitrates[ridx];
                        if (rate == srate->bitrate) {
                                mask |= 1 << ridx;
                                break;
                        }
                }
                if (ridx == sband->n_bitrates)
                        return 0; /* rate not found */
        }

        return mask;
}

static bool ht_rateset_to_mask(struct ieee80211_supported_band *sband,
                               u8 *rates, u8 rates_len,
                               u8 mcs[IEEE80211_HT_MCS_MASK_LEN])
{
        u8 i;

        memset(mcs, 0, IEEE80211_HT_MCS_MASK_LEN);

        for (i = 0; i < rates_len; i++) {
                int ridx, rbit;

                ridx = rates[i] / 8;
                rbit = BIT(rates[i] % 8);

                /* check validity */
                if ((ridx < 0) || (ridx >= IEEE80211_HT_MCS_MASK_LEN))
                        return false;

                /* check availability */
                ridx = array_index_nospec(ridx, IEEE80211_HT_MCS_MASK_LEN);
                if (sband->ht_cap.mcs.rx_mask[ridx] & rbit)
                        mcs[ridx] |= rbit;
                else
                        return false;
        }

        return true;
}

static u16 vht_mcs_map_to_mcs_mask(u8 vht_mcs_map)
{
        u16 mcs_mask = 0;

        switch (vht_mcs_map) {
        case IEEE80211_VHT_MCS_NOT_SUPPORTED:
                break;
        case IEEE80211_VHT_MCS_SUPPORT_0_7:
                mcs_mask = 0x00FF;
                break;
        case IEEE80211_VHT_MCS_SUPPORT_0_8:
                mcs_mask = 0x01FF;
                break;
        case IEEE80211_VHT_MCS_SUPPORT_0_9:
                mcs_mask = 0x03FF;
                break;
        default:
                break;
        }

        return mcs_mask;
}

static void vht_build_mcs_mask(u16 vht_mcs_map,
                               u16 vht_mcs_mask[NL80211_VHT_NSS_MAX])
{
        u8 nss;

        for (nss = 0; nss < NL80211_VHT_NSS_MAX; nss++) {
                vht_mcs_mask[nss] = vht_mcs_map_to_mcs_mask(vht_mcs_map & 0x03);
                vht_mcs_map >>= 2;
        }
}

static bool vht_set_mcs_mask(struct ieee80211_supported_band *sband,
                             struct nl80211_txrate_vht *txrate,
                             u16 mcs[NL80211_VHT_NSS_MAX])
{
        u16 tx_mcs_map = le16_to_cpu(sband->vht_cap.vht_mcs.tx_mcs_map);
        u16 tx_mcs_mask[NL80211_VHT_NSS_MAX] = {};
        u8 i;

        if (!sband->vht_cap.vht_supported)
                return false;

        memset(mcs, 0, sizeof(u16) * NL80211_VHT_NSS_MAX);

        /* Build vht_mcs_mask from VHT capabilities */
        vht_build_mcs_mask(tx_mcs_map, tx_mcs_mask);

        for (i = 0; i < NL80211_VHT_NSS_MAX; i++) {
                if ((tx_mcs_mask[i] & txrate->mcs[i]) == txrate->mcs[i])
                        mcs[i] = txrate->mcs[i];
                else
                        return false;
        }

        return true;
}

static u16 he_mcs_map_to_mcs_mask(u8 he_mcs_map)
{
        switch (he_mcs_map) {
        case IEEE80211_HE_MCS_NOT_SUPPORTED:
                return 0;
        case IEEE80211_HE_MCS_SUPPORT_0_7:
                return 0x00FF;
        case IEEE80211_HE_MCS_SUPPORT_0_9:
                return 0x03FF;
        case IEEE80211_HE_MCS_SUPPORT_0_11:
                return 0xFFF;
        default:
                break;
        }
        return 0;
}

static void he_build_mcs_mask(u16 he_mcs_map,
                              u16 he_mcs_mask[NL80211_HE_NSS_MAX])
{
        u8 nss;

        for (nss = 0; nss < NL80211_HE_NSS_MAX; nss++) {
                he_mcs_mask[nss] = he_mcs_map_to_mcs_mask(he_mcs_map & 0x03);
                he_mcs_map >>= 2;
        }
}

static u16 he_get_txmcsmap(struct genl_info *info, unsigned int link_id,
                           const struct ieee80211_sta_he_cap *he_cap)
{
        struct net_device *dev = info->user_ptr[1];
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_chan_def *chandef;
        __le16 tx_mcs;

        chandef = wdev_chandef(wdev, link_id);
        if (!chandef) {
                /*
                 * This is probably broken, but we never maintained
                 * a chandef in these cases, so it always was.
                 */
                return le16_to_cpu(he_cap->he_mcs_nss_supp.tx_mcs_80);
        }

        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_80P80:
                tx_mcs = he_cap->he_mcs_nss_supp.tx_mcs_80p80;
                break;
        case NL80211_CHAN_WIDTH_160:
                tx_mcs = he_cap->he_mcs_nss_supp.tx_mcs_160;
                break;
        default:
                tx_mcs = he_cap->he_mcs_nss_supp.tx_mcs_80;
                break;
        }

        return le16_to_cpu(tx_mcs);
}

static bool he_set_mcs_mask(struct genl_info *info,
                            struct wireless_dev *wdev,
                            struct ieee80211_supported_band *sband,
                            struct nl80211_txrate_he *txrate,
                            u16 mcs[NL80211_HE_NSS_MAX],
                            unsigned int link_id)
{
        const struct ieee80211_sta_he_cap *he_cap;
        u16 tx_mcs_mask[NL80211_HE_NSS_MAX] = {};
        u16 tx_mcs_map = 0;
        u8 i;

        he_cap = ieee80211_get_he_iftype_cap(sband, wdev->iftype);
        if (!he_cap)
                return false;

        memset(mcs, 0, sizeof(u16) * NL80211_HE_NSS_MAX);

        tx_mcs_map = he_get_txmcsmap(info, link_id, he_cap);

        /* Build he_mcs_mask from HE capabilities */
        he_build_mcs_mask(tx_mcs_map, tx_mcs_mask);

        for (i = 0; i < NL80211_HE_NSS_MAX; i++) {
                if ((tx_mcs_mask[i] & txrate->mcs[i]) == txrate->mcs[i])
                        mcs[i] = txrate->mcs[i];
                else
                        return false;
        }

        return true;
}

static void eht_build_mcs_mask(struct genl_info *info,
                               const struct ieee80211_sta_eht_cap *eht_cap,
                               u8 mcs_nss_len, u16 *mcs_mask)
{
        struct net_device *dev = info->user_ptr[1];
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        u8 nss, mcs_7 = 0, mcs_9 = 0, mcs_11 = 0, mcs_13 = 0;
        unsigned int link_id = nl80211_link_id(info->attrs);

        if (mcs_nss_len == 4) {
                const struct ieee80211_eht_mcs_nss_supp_20mhz_only *mcs =
                                        &eht_cap->eht_mcs_nss_supp.only_20mhz;

                mcs_7 = u8_get_bits(mcs->rx_tx_mcs7_max_nss,
                                    IEEE80211_EHT_MCS_NSS_TX);
                mcs_9 = u8_get_bits(mcs->rx_tx_mcs9_max_nss,
                                    IEEE80211_EHT_MCS_NSS_TX);
                mcs_11 = u8_get_bits(mcs->rx_tx_mcs11_max_nss,
                                     IEEE80211_EHT_MCS_NSS_TX);
                mcs_13 = u8_get_bits(mcs->rx_tx_mcs13_max_nss,
                                     IEEE80211_EHT_MCS_NSS_TX);

        } else {
                const struct ieee80211_eht_mcs_nss_supp_bw *mcs;
                enum nl80211_chan_width width;

                switch (wdev->iftype) {
                case NL80211_IFTYPE_ADHOC:
                        width = wdev->u.ibss.chandef.width;
                        break;
                case NL80211_IFTYPE_MESH_POINT:
                        width = wdev->u.mesh.chandef.width;
                        break;
                case NL80211_IFTYPE_OCB:
                        width = wdev->u.ocb.chandef.width;
                        break;
                default:
                        if (wdev->valid_links)
                                width = wdev->links[link_id].ap.chandef.width;
                        else
                                width = wdev->u.ap.preset_chandef.width;
                        break;
                }

                switch (width) {
                case NL80211_CHAN_WIDTH_320:
                        mcs = &eht_cap->eht_mcs_nss_supp.bw._320;
                        break;
                case NL80211_CHAN_WIDTH_160:
                        mcs = &eht_cap->eht_mcs_nss_supp.bw._160;
                        break;
                default:
                        mcs = &eht_cap->eht_mcs_nss_supp.bw._80;
                        break;
                }

                mcs_7 = u8_get_bits(mcs->rx_tx_mcs9_max_nss,
                                    IEEE80211_EHT_MCS_NSS_TX);
                mcs_9 = u8_get_bits(mcs->rx_tx_mcs9_max_nss,
                                    IEEE80211_EHT_MCS_NSS_TX);
                mcs_11 = u8_get_bits(mcs->rx_tx_mcs11_max_nss,
                                     IEEE80211_EHT_MCS_NSS_TX);
                mcs_13 = u8_get_bits(mcs->rx_tx_mcs13_max_nss,
                                     IEEE80211_EHT_MCS_NSS_TX);
        }

        /* Enable MCS 14 for NSS 0 */
        if (eht_cap->eht_cap_elem.phy_cap_info[6] &
            IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP)
                mcs_mask[0] |= 0x4000;

        /* Enable MCS 15 for NSS 0 */
        mcs_mask[0] |= 0x8000;

        for (nss = 0; nss < NL80211_EHT_NSS_MAX; nss++) {
                if (!mcs_7)
                        continue;
                mcs_mask[nss] |= 0x00FF;
                mcs_7--;

                if (!mcs_9)
                        continue;
                mcs_mask[nss] |= 0x0300;
                mcs_9--;

                if (!mcs_11)
                        continue;
                mcs_mask[nss] |= 0x0C00;
                mcs_11--;

                if (!mcs_13)
                        continue;
                mcs_mask[nss] |= 0x3000;
                mcs_13--;
        }
}

static bool eht_set_mcs_mask(struct genl_info *info, struct wireless_dev *wdev,
                             struct ieee80211_supported_band *sband,
                             struct nl80211_txrate_eht *txrate,
                             u16 mcs[NL80211_EHT_NSS_MAX])
{
        const struct ieee80211_sta_he_cap *he_cap;
        const struct ieee80211_sta_eht_cap *eht_cap;
        u16 tx_mcs_mask[NL80211_EHT_NSS_MAX] = { 0 };
        u8 i, mcs_nss_len;

        he_cap = ieee80211_get_he_iftype_cap(sband, wdev->iftype);
        if (!he_cap)
                return false;

        eht_cap = ieee80211_get_eht_iftype_cap(sband, wdev->iftype);
        if (!eht_cap)
                return false;

        /* Checks for MCS 14 */
        if (txrate->mcs[0] & 0x4000) {
                if (sband->band != NL80211_BAND_6GHZ)
                        return false;

                if (!(eht_cap->eht_cap_elem.phy_cap_info[6] &
                      IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP))
                        return false;
        }

        mcs_nss_len = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem,
                                                 &eht_cap->eht_cap_elem,
                                                 wdev->iftype ==
                                                        NL80211_IFTYPE_STATION);

        if (mcs_nss_len == 3) {
                /* Supported iftypes for setting non-20 MHZ only EHT MCS */
                switch (wdev->iftype) {
                case NL80211_IFTYPE_ADHOC:
                case NL80211_IFTYPE_AP:
                case NL80211_IFTYPE_P2P_GO:
                case NL80211_IFTYPE_MESH_POINT:
                case NL80211_IFTYPE_OCB:
                        break;
                default:
                        return false;
                }
        }

        /* Build eht_mcs_mask from EHT and HE capabilities */
        eht_build_mcs_mask(info, eht_cap, mcs_nss_len, tx_mcs_mask);

        memset(mcs, 0, sizeof(u16) * NL80211_EHT_NSS_MAX);
        for (i = 0; i < NL80211_EHT_NSS_MAX; i++) {
                if ((tx_mcs_mask[i] & txrate->mcs[i]) == txrate->mcs[i])
                        mcs[i] = txrate->mcs[i];
                else
                        return false;
        }

        return true;
}

static int nl80211_parse_tx_bitrate_mask(struct genl_info *info,
                                         struct nlattr *attrs[],
                                         enum nl80211_attrs attr,
                                         struct cfg80211_bitrate_mask *mask,
                                         struct net_device *dev,
                                         bool default_all_enabled,
                                         unsigned int link_id)
{
        struct nlattr *tb[NL80211_TXRATE_MAX + 1];
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        int rem, i;
        struct nlattr *tx_rates;
        struct ieee80211_supported_band *sband;
        u16 vht_tx_mcs_map, he_tx_mcs_map;

        memset(mask, 0, sizeof(*mask));
        /* Default to all rates enabled */
        for (i = 0; i < NUM_NL80211_BANDS; i++) {
                const struct ieee80211_sta_he_cap *he_cap;
                const struct ieee80211_sta_eht_cap *eht_cap;
                u8 mcs_nss_len;

                if (!default_all_enabled)
                        break;

                sband = rdev->wiphy.bands[i];

                if (!sband)
                        continue;

                mask->control[i].legacy = (1 << sband->n_bitrates) - 1;
                memcpy(mask->control[i].ht_mcs,
                       sband->ht_cap.mcs.rx_mask,
                       sizeof(mask->control[i].ht_mcs));

                if (sband->vht_cap.vht_supported) {
                        vht_tx_mcs_map = le16_to_cpu(sband->vht_cap.vht_mcs.tx_mcs_map);
                        vht_build_mcs_mask(vht_tx_mcs_map, mask->control[i].vht_mcs);
                }

                he_cap = ieee80211_get_he_iftype_cap(sband, wdev->iftype);
                if (!he_cap)
                        continue;

                he_tx_mcs_map = he_get_txmcsmap(info, link_id, he_cap);
                he_build_mcs_mask(he_tx_mcs_map, mask->control[i].he_mcs);

                mask->control[i].he_gi = 0xFF;
                mask->control[i].he_ltf = 0xFF;

                eht_cap = ieee80211_get_eht_iftype_cap(sband, wdev->iftype);
                if (!eht_cap)
                        continue;

                mcs_nss_len = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem,
                                                         &eht_cap->eht_cap_elem,
                                                         wdev->iftype ==
                                                         NL80211_IFTYPE_STATION);

                eht_build_mcs_mask(info, eht_cap, mcs_nss_len,
                                   mask->control[i].eht_mcs);

                mask->control[i].eht_gi = 0xFF;
                mask->control[i].eht_ltf = 0xFF;
        }

        /* if no rates are given set it back to the defaults */
        if (!attrs[attr])
                goto out;

        /* The nested attribute uses enum nl80211_band as the index. This maps
         * directly to the enum nl80211_band values used in cfg80211.
         */
        BUILD_BUG_ON(NL80211_MAX_SUPP_HT_RATES > IEEE80211_HT_MCS_MASK_LEN * 8);
        nla_for_each_nested(tx_rates, attrs[attr], rem) {
                int band = nla_type(tx_rates);
                int err;

                if (band < 0 || band >= NUM_NL80211_BANDS)
                        return -EINVAL;
                sband = rdev->wiphy.bands[band];
                if (sband == NULL)
                        return -EINVAL;
                err = nla_parse_nested_deprecated(tb, NL80211_TXRATE_MAX,
                                                  tx_rates,
                                                  nl80211_txattr_policy,
                                                  info->extack);
                if (err)
                        return err;
                if (tb[NL80211_TXRATE_LEGACY]) {
                        mask->control[band].legacy = rateset_to_mask(
                                sband,
                                nla_data(tb[NL80211_TXRATE_LEGACY]),
                                nla_len(tb[NL80211_TXRATE_LEGACY]));
                        if ((mask->control[band].legacy == 0) &&
                            nla_len(tb[NL80211_TXRATE_LEGACY]))
                                return -EINVAL;
                }
                if (tb[NL80211_TXRATE_HT]) {
                        if (!ht_rateset_to_mask(
                                        sband,
                                        nla_data(tb[NL80211_TXRATE_HT]),
                                        nla_len(tb[NL80211_TXRATE_HT]),
                                        mask->control[band].ht_mcs))
                                return -EINVAL;
                }

                if (tb[NL80211_TXRATE_VHT]) {
                        if (!vht_set_mcs_mask(
                                        sband,
                                        nla_data(tb[NL80211_TXRATE_VHT]),
                                        mask->control[band].vht_mcs))
                                return -EINVAL;
                }

                if (tb[NL80211_TXRATE_GI]) {
                        mask->control[band].gi =
                                nla_get_u8(tb[NL80211_TXRATE_GI]);
                        if (mask->control[band].gi > NL80211_TXRATE_FORCE_LGI)
                                return -EINVAL;
                }
                if (tb[NL80211_TXRATE_HE] &&
                    !he_set_mcs_mask(info, wdev, sband,
                                     nla_data(tb[NL80211_TXRATE_HE]),
                                     mask->control[band].he_mcs,
                                     link_id))
                        return -EINVAL;

                if (tb[NL80211_TXRATE_HE_GI])
                        mask->control[band].he_gi =
                                nla_get_u8(tb[NL80211_TXRATE_HE_GI]);
                if (tb[NL80211_TXRATE_HE_LTF])
                        mask->control[band].he_ltf =
                                nla_get_u8(tb[NL80211_TXRATE_HE_LTF]);

                if (tb[NL80211_TXRATE_EHT] &&
                    !eht_set_mcs_mask(info, wdev, sband,
                                      nla_data(tb[NL80211_TXRATE_EHT]),
                                      mask->control[band].eht_mcs))
                        return -EINVAL;

                if (tb[NL80211_TXRATE_EHT_GI])
                        mask->control[band].eht_gi =
                                nla_get_u8(tb[NL80211_TXRATE_EHT_GI]);
                if (tb[NL80211_TXRATE_EHT_LTF])
                        mask->control[band].eht_ltf =
                                nla_get_u8(tb[NL80211_TXRATE_EHT_LTF]);

                if (mask->control[band].legacy == 0) {
                        /* don't allow empty legacy rates if HT, VHT, HE or EHT
                         * are not even supported.
                         */
                        if (!(rdev->wiphy.bands[band]->ht_cap.ht_supported ||
                              rdev->wiphy.bands[band]->vht_cap.vht_supported ||
                              ieee80211_get_he_iftype_cap(sband, wdev->iftype) ||
                              ieee80211_get_eht_iftype_cap(sband, wdev->iftype)))
                                return -EINVAL;

                        for (i = 0; i < IEEE80211_HT_MCS_MASK_LEN; i++)
                                if (mask->control[band].ht_mcs[i])
                                        goto out;

                        for (i = 0; i < NL80211_VHT_NSS_MAX; i++)
                                if (mask->control[band].vht_mcs[i])
                                        goto out;

                        for (i = 0; i < NL80211_HE_NSS_MAX; i++)
                                if (mask->control[band].he_mcs[i])
                                        goto out;

                        for (i = 0; i < NL80211_EHT_NSS_MAX; i++)
                                if (mask->control[band].eht_mcs[i])
                                        goto out;

                        /* legacy and mcs rates may not be both empty */
                        return -EINVAL;
                }
        }

out:
        return 0;
}

static int validate_beacon_tx_rate(struct cfg80211_registered_device *rdev,
                                   enum nl80211_band band,
                                   struct cfg80211_bitrate_mask *beacon_rate)
{
        u32 count_ht, count_vht, count_he, count_eht, i;
        u32 rate = beacon_rate->control[band].legacy;

        /* Allow only one rate */
        if (hweight32(rate) > 1)
                return -EINVAL;

        count_ht = 0;
        for (i = 0; i < IEEE80211_HT_MCS_MASK_LEN; i++) {
                if (hweight8(beacon_rate->control[band].ht_mcs[i]) > 1) {
                        return -EINVAL;
                } else if (beacon_rate->control[band].ht_mcs[i]) {
                        count_ht++;
                        if (count_ht > 1)
                                return -EINVAL;
                }
                if (count_ht && rate)
                        return -EINVAL;
        }

        count_vht = 0;
        for (i = 0; i < NL80211_VHT_NSS_MAX; i++) {
                if (hweight16(beacon_rate->control[band].vht_mcs[i]) > 1) {
                        return -EINVAL;
                } else if (beacon_rate->control[band].vht_mcs[i]) {
                        count_vht++;
                        if (count_vht > 1)
                                return -EINVAL;
                }
                if (count_vht && rate)
                        return -EINVAL;
        }

        count_he = 0;
        for (i = 0; i < NL80211_HE_NSS_MAX; i++) {
                if (hweight16(beacon_rate->control[band].he_mcs[i]) > 1) {
                        return -EINVAL;
                } else if (beacon_rate->control[band].he_mcs[i]) {
                        count_he++;
                        if (count_he > 1)
                                return -EINVAL;
                }
                if (count_he && rate)
                        return -EINVAL;
        }

        count_eht = 0;
        for (i = 0; i < NL80211_EHT_NSS_MAX; i++) {
                if (hweight16(beacon_rate->control[band].eht_mcs[i]) > 1) {
                        return -EINVAL;
                } else if (beacon_rate->control[band].eht_mcs[i]) {
                        count_eht++;
                        if (count_eht > 1)
                                return -EINVAL;
                }
                if (count_eht && rate)
                        return -EINVAL;
        }

        if ((count_ht && count_vht && count_he && count_eht) ||
            (!rate && !count_ht && !count_vht && !count_he && !count_eht))
                return -EINVAL;

        if (rate &&
            !wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_BEACON_RATE_LEGACY))
                return -EINVAL;
        if (count_ht &&
            !wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_BEACON_RATE_HT))
                return -EINVAL;
        if (count_vht &&
            !wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_BEACON_RATE_VHT))
                return -EINVAL;
        if (count_he &&
            !wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_BEACON_RATE_HE))
                return -EINVAL;

        if (count_eht &&
            !wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_BEACON_RATE_EHT))
                return -EINVAL;

        return 0;
}

static int nl80211_parse_mbssid_config(struct wiphy *wiphy,
                                       struct net_device *dev,
                                       unsigned int link_id,
                                       struct nlattr *attrs,
                                       struct cfg80211_mbssid_config *config,
                                       u8 num_elems)
{
        struct nlattr *tb[NL80211_MBSSID_CONFIG_ATTR_MAX + 1];
        int tx_link_id = -1;

        if (!wiphy->mbssid_max_interfaces)
                return -EOPNOTSUPP;

        if (nla_parse_nested(tb, NL80211_MBSSID_CONFIG_ATTR_MAX, attrs, NULL,
                             NULL) ||
            !tb[NL80211_MBSSID_CONFIG_ATTR_INDEX])
                return -EINVAL;

        config->ema = nla_get_flag(tb[NL80211_MBSSID_CONFIG_ATTR_EMA]);
        if (config->ema) {
                if (!wiphy->ema_max_profile_periodicity)
                        return -EOPNOTSUPP;

                if (num_elems > wiphy->ema_max_profile_periodicity)
                        return -EINVAL;
        }

        config->index = nla_get_u8(tb[NL80211_MBSSID_CONFIG_ATTR_INDEX]);
        if (config->index >= wiphy->mbssid_max_interfaces ||
            (!config->index && !num_elems))
                return -EINVAL;

        if (tb[NL80211_MBSSID_CONFIG_ATTR_TX_LINK_ID])
                tx_link_id = nla_get_u8(tb[NL80211_MBSSID_CONFIG_ATTR_TX_LINK_ID]);

        if (tb[NL80211_MBSSID_CONFIG_ATTR_TX_IFINDEX]) {
                u32 tx_ifindex =
                        nla_get_u32(tb[NL80211_MBSSID_CONFIG_ATTR_TX_IFINDEX]);

                if ((!config->index && tx_ifindex != dev->ifindex) ||
                    (config->index && tx_ifindex == dev->ifindex))
                        return -EINVAL;

                if (tx_ifindex != dev->ifindex) {
                        struct net_device *tx_netdev =
                                dev_get_by_index(wiphy_net(wiphy), tx_ifindex);

                        if (!tx_netdev || !tx_netdev->ieee80211_ptr ||
                            tx_netdev->ieee80211_ptr->wiphy != wiphy ||
                            tx_netdev->ieee80211_ptr->iftype !=
                                                        NL80211_IFTYPE_AP) {
                                dev_put(tx_netdev);
                                return -EINVAL;
                        }

                        config->tx_wdev = tx_netdev->ieee80211_ptr;
                        /* Caller should call dev_put(config->tx_wdev) from this point */

                        if (config->tx_wdev->valid_links) {
                                if (tx_link_id == -1 ||
                                    !(config->tx_wdev->valid_links & BIT(tx_link_id)))
                                        return -ENOLINK;

                                config->tx_link_id = tx_link_id;
                        }
                } else {
                        if (tx_link_id >= 0 && tx_link_id != link_id)
                                return -EINVAL;

                        config->tx_wdev = dev->ieee80211_ptr;
                }
        } else if (!config->index) {
                if (tx_link_id >= 0 && tx_link_id != link_id)
                        return -EINVAL;

                config->tx_wdev = dev->ieee80211_ptr;
        } else {
                return -EINVAL;
        }

        return 0;
}

static struct cfg80211_mbssid_elems *
nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs,
                           struct netlink_ext_ack *extack)
{
        struct nlattr *nl_elems;
        struct cfg80211_mbssid_elems *elems;
        int rem_elems;
        u8 i = 0, num_elems = 0;

        if (!wiphy->mbssid_max_interfaces)
                return ERR_PTR(-EINVAL);

        nla_for_each_nested(nl_elems, attrs, rem_elems) {
                int ret;

                ret = validate_ie_attr(nl_elems, extack);
                if (ret)
                        return ERR_PTR(ret);

                if (num_elems >= 255)
                        return ERR_PTR(-EINVAL);
                num_elems++;
        }

        elems = kzalloc_flex(*elems, elem, num_elems);
        if (!elems)
                return ERR_PTR(-ENOMEM);
        elems->cnt = num_elems;

        nla_for_each_nested(nl_elems, attrs, rem_elems) {
                elems->elem[i].data = nla_data(nl_elems);
                elems->elem[i].len = nla_len(nl_elems);
                i++;
        }
        return elems;
}

static struct cfg80211_rnr_elems *
nl80211_parse_rnr_elems(struct wiphy *wiphy, struct nlattr *attrs,
                        struct netlink_ext_ack *extack)
{
        struct nlattr *nl_elems;
        struct cfg80211_rnr_elems *elems;
        int rem_elems;
        u8 i = 0, num_elems = 0;

        nla_for_each_nested(nl_elems, attrs, rem_elems) {
                int ret;

                ret = validate_ie_attr(nl_elems, extack);
                if (ret)
                        return ERR_PTR(ret);

                if (num_elems >= 255)
                        return ERR_PTR(-EINVAL);

                num_elems++;
        }

        elems = kzalloc_flex(*elems, elem, num_elems);
        if (!elems)
                return ERR_PTR(-ENOMEM);
        elems->cnt = num_elems;

        nla_for_each_nested(nl_elems, attrs, rem_elems) {
                elems->elem[i].data = nla_data(nl_elems);
                elems->elem[i].len = nla_len(nl_elems);
                i++;
        }
        return elems;
}

static int nl80211_parse_he_bss_color(struct nlattr *attrs,
                                      struct cfg80211_he_bss_color *he_bss_color)
{
        struct nlattr *tb[NL80211_HE_BSS_COLOR_ATTR_MAX + 1];
        int err;

        err = nla_parse_nested(tb, NL80211_HE_BSS_COLOR_ATTR_MAX, attrs,
                               he_bss_color_policy, NULL);
        if (err)
                return err;

        if (!tb[NL80211_HE_BSS_COLOR_ATTR_COLOR])
                return -EINVAL;

        he_bss_color->color =
                nla_get_u8(tb[NL80211_HE_BSS_COLOR_ATTR_COLOR]);
        he_bss_color->enabled =
                !nla_get_flag(tb[NL80211_HE_BSS_COLOR_ATTR_DISABLED]);
        he_bss_color->partial =
                nla_get_flag(tb[NL80211_HE_BSS_COLOR_ATTR_PARTIAL]);

        return 0;
}

static void nl80211_check_ap_rate_selectors(struct cfg80211_beacon_data *bcn,
                                            const struct element *rates)
{
        int i;

        if (!rates)
                return;

        for (i = 0; i < rates->datalen; i++) {
                if (rates->data[i] == BSS_MEMBERSHIP_SELECTOR_HT_PHY)
                        bcn->ht_required = true;
                if (rates->data[i] == BSS_MEMBERSHIP_SELECTOR_VHT_PHY)
                        bcn->vht_required = true;
        }
}

/*
 * Since the nl80211 API didn't include, from the beginning, attributes about
 * HT/VHT/... operation, we parse them out of the elements and check for
 * validity for use by drivers/mac80211.
 */
static int nl80211_calculate_ap_operation(struct nlattr *attrs[],
                                          struct cfg80211_beacon_data *bcn,
                                          struct netlink_ext_ack *extack)
{
        size_t ies_len = bcn->tail_len;
        const u8 *ies = bcn->tail;
        const struct element *rates;
        const struct element *op;

        rates = cfg80211_find_elem(WLAN_EID_SUPP_RATES, ies, ies_len);
        nl80211_check_ap_rate_selectors(bcn, rates);

        rates = cfg80211_find_elem(WLAN_EID_EXT_SUPP_RATES, ies, ies_len);
        nl80211_check_ap_rate_selectors(bcn, rates);

        op = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION, ies, ies_len);
        if (op) {
                if (op->datalen < sizeof(*bcn->he_oper) + 1) {
                        NL_SET_ERR_MSG(extack, "bad HE operation in beacon");
                        return -EINVAL;
                }
                bcn->he_oper = (void *)(op->data + 1);
                /* takes extension ID into account */
                if (op->datalen < ieee80211_he_oper_size((void *)bcn->he_oper)) {
                        NL_SET_ERR_MSG(extack, "bad HE operation in beacon");
                        return -EINVAL;
                }
        }

        op = cfg80211_find_elem(WLAN_EID_HT_OPERATION, ies, ies_len);
        if (op) {
                if (op->datalen < sizeof(*bcn->ht_oper)) {
                        NL_SET_ERR_MSG(extack, "bad HT operation in beacon");
                        return -EINVAL;
                }
                bcn->ht_oper = (void *)op->data;
        }

        op = cfg80211_find_elem(WLAN_EID_VHT_OPERATION, ies, ies_len);
        if (op) {
                if (op->datalen < sizeof(*bcn->vht_oper)) {
                        NL_SET_ERR_MSG(extack, "bad VHT operation in beacon");
                        return -EINVAL;
                }
                bcn->vht_oper = (void *)op->data;
        }

        op = cfg80211_find_ext_elem(WLAN_EID_EXT_EHT_OPERATION, ies, ies_len);
        if (op) {
                if (!ieee80211_eht_oper_size_ok(op->data + 1,
                                                op->datalen - 1)) {
                        NL_SET_ERR_MSG(extack, "bad EHT operation in beacon");
                        return -EINVAL;
                }
                bcn->eht_oper = (void *)(op->data + 1);
        }

        op = cfg80211_find_ext_elem(WLAN_EID_EXT_UHR_OPER, ies, ies_len);
        if (op) {
                /* need full UHR operation separately */
                if (!attrs[NL80211_ATTR_UHR_OPERATION]) {
                        NL_SET_ERR_MSG(extack, "missing UHR operation");
                        return -EINVAL;
                }
                bcn->uhr_oper = nla_data(attrs[NL80211_ATTR_UHR_OPERATION]);
        } else if (attrs[NL80211_ATTR_UHR_OPERATION]) {
                NL_SET_ERR_MSG(extack, "unexpected UHR operation");
                return -EINVAL;
        }

        return 0;
}

static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev,
                                struct nlattr *attrs[],
                                struct cfg80211_beacon_data *bcn,
                                struct ieee80211_channel *chan,
                                struct netlink_ext_ack *extack)
{
        bool haveinfo = false;
        int err;

        memset(bcn, 0, sizeof(*bcn));

        bcn->link_id = nl80211_link_id(attrs);

        if (attrs[NL80211_ATTR_BEACON_HEAD]) {
                bcn->head = nla_data(attrs[NL80211_ATTR_BEACON_HEAD]);
                bcn->head_len = nla_len(attrs[NL80211_ATTR_BEACON_HEAD]);
                if (!bcn->head_len)
                        return -EINVAL;
                haveinfo = true;
        }

        if (attrs[NL80211_ATTR_BEACON_TAIL]) {
                bcn->tail = nla_data(attrs[NL80211_ATTR_BEACON_TAIL]);
                bcn->tail_len = nla_len(attrs[NL80211_ATTR_BEACON_TAIL]);
                haveinfo = true;
        }

        if (!haveinfo)
                return -EINVAL;

        if (attrs[NL80211_ATTR_IE]) {
                bcn->beacon_ies = nla_data(attrs[NL80211_ATTR_IE]);
                bcn->beacon_ies_len = nla_len(attrs[NL80211_ATTR_IE]);
        }

        if (attrs[NL80211_ATTR_IE_PROBE_RESP]) {
                bcn->proberesp_ies =
                        nla_data(attrs[NL80211_ATTR_IE_PROBE_RESP]);
                bcn->proberesp_ies_len =
                        nla_len(attrs[NL80211_ATTR_IE_PROBE_RESP]);
        }

        if (attrs[NL80211_ATTR_IE_ASSOC_RESP]) {
                bcn->assocresp_ies =
                        nla_data(attrs[NL80211_ATTR_IE_ASSOC_RESP]);
                bcn->assocresp_ies_len =
                        nla_len(attrs[NL80211_ATTR_IE_ASSOC_RESP]);
        }

        if (attrs[NL80211_ATTR_PROBE_RESP]) {
                bcn->probe_resp = nla_data(attrs[NL80211_ATTR_PROBE_RESP]);
                bcn->probe_resp_len = nla_len(attrs[NL80211_ATTR_PROBE_RESP]);
        }

        if (attrs[NL80211_ATTR_FTM_RESPONDER]) {
                struct nlattr *tb[NL80211_FTM_RESP_ATTR_MAX + 1];

                err = nla_parse_nested_deprecated(tb,
                                                  NL80211_FTM_RESP_ATTR_MAX,
                                                  attrs[NL80211_ATTR_FTM_RESPONDER],
                                                  NULL, NULL);
                if (err)
                        return err;

                if (tb[NL80211_FTM_RESP_ATTR_ENABLED] &&
                    wiphy_ext_feature_isset(&rdev->wiphy,
                                            NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER))
                        bcn->ftm_responder = 1;
                else
                        return -EOPNOTSUPP;

                if (tb[NL80211_FTM_RESP_ATTR_LCI]) {
                        bcn->lci = nla_data(tb[NL80211_FTM_RESP_ATTR_LCI]);
                        bcn->lci_len = nla_len(tb[NL80211_FTM_RESP_ATTR_LCI]);
                }

                if (tb[NL80211_FTM_RESP_ATTR_CIVICLOC]) {
                        bcn->civicloc = nla_data(tb[NL80211_FTM_RESP_ATTR_CIVICLOC]);
                        bcn->civicloc_len = nla_len(tb[NL80211_FTM_RESP_ATTR_CIVICLOC]);
                }
        } else {
                bcn->ftm_responder = -1;
        }

        if (attrs[NL80211_ATTR_HE_BSS_COLOR]) {
                err = nl80211_parse_he_bss_color(attrs[NL80211_ATTR_HE_BSS_COLOR],
                                                 &bcn->he_bss_color);
                if (err)
                        return err;
                bcn->he_bss_color_valid = true;
        }

        if (attrs[NL80211_ATTR_MBSSID_ELEMS]) {
                struct cfg80211_mbssid_elems *mbssid =
                        nl80211_parse_mbssid_elems(&rdev->wiphy,
                                                   attrs[NL80211_ATTR_MBSSID_ELEMS],
                                                   extack);

                if (IS_ERR(mbssid))
                        return PTR_ERR(mbssid);

                bcn->mbssid_ies = mbssid;

                if (bcn->mbssid_ies && attrs[NL80211_ATTR_EMA_RNR_ELEMS]) {
                        struct cfg80211_rnr_elems *rnr =
                                nl80211_parse_rnr_elems(&rdev->wiphy,
                                                        attrs[NL80211_ATTR_EMA_RNR_ELEMS],
                                                        extack);

                        if (IS_ERR(rnr))
                                return PTR_ERR(rnr);

                        if (rnr && rnr->cnt < bcn->mbssid_ies->cnt) {
                                kfree(rnr);
                                return -EINVAL;
                        }

                        bcn->rnr_ies = rnr;
                }
        }

        err = nl80211_calculate_ap_operation(attrs, bcn, extack);
        if (err)
                return err;

        if (bcn->he_oper && (chan->flags & IEEE80211_CHAN_NO_HE))
                return -EOPNOTSUPP;

        if (bcn->eht_oper && (chan->flags & IEEE80211_CHAN_NO_EHT))
                return -EOPNOTSUPP;

        if (bcn->uhr_oper && (chan->flags & IEEE80211_CHAN_NO_UHR))
                return -EOPNOTSUPP;

        return 0;
}

static int nl80211_parse_he_obss_pd(struct nlattr *attrs,
                                    struct ieee80211_he_obss_pd *he_obss_pd)
{
        struct nlattr *tb[NL80211_HE_OBSS_PD_ATTR_MAX + 1];
        int err;

        err = nla_parse_nested(tb, NL80211_HE_OBSS_PD_ATTR_MAX, attrs,
                               he_obss_pd_policy, NULL);
        if (err)
                return err;

        if (!tb[NL80211_HE_OBSS_PD_ATTR_SR_CTRL])
                return -EINVAL;

        he_obss_pd->sr_ctrl = nla_get_u8(tb[NL80211_HE_OBSS_PD_ATTR_SR_CTRL]);

        if (tb[NL80211_HE_OBSS_PD_ATTR_MIN_OFFSET])
                he_obss_pd->min_offset =
                        nla_get_u8(tb[NL80211_HE_OBSS_PD_ATTR_MIN_OFFSET]);
        if (tb[NL80211_HE_OBSS_PD_ATTR_MAX_OFFSET])
                he_obss_pd->max_offset =
                        nla_get_u8(tb[NL80211_HE_OBSS_PD_ATTR_MAX_OFFSET]);
        if (tb[NL80211_HE_OBSS_PD_ATTR_NON_SRG_MAX_OFFSET])
                he_obss_pd->non_srg_max_offset =
                        nla_get_u8(tb[NL80211_HE_OBSS_PD_ATTR_NON_SRG_MAX_OFFSET]);

        if (he_obss_pd->min_offset > he_obss_pd->max_offset)
                return -EINVAL;

        if (tb[NL80211_HE_OBSS_PD_ATTR_BSS_COLOR_BITMAP])
                memcpy(he_obss_pd->bss_color_bitmap,
                       nla_data(tb[NL80211_HE_OBSS_PD_ATTR_BSS_COLOR_BITMAP]),
                       sizeof(he_obss_pd->bss_color_bitmap));

        if (tb[NL80211_HE_OBSS_PD_ATTR_PARTIAL_BSSID_BITMAP])
                memcpy(he_obss_pd->partial_bssid_bitmap,
                       nla_data(tb[NL80211_HE_OBSS_PD_ATTR_PARTIAL_BSSID_BITMAP]),
                       sizeof(he_obss_pd->partial_bssid_bitmap));

        he_obss_pd->enable = true;

        return 0;
}

static int nl80211_parse_fils_discovery(struct cfg80211_registered_device *rdev,
                                        struct nlattr *attrs,
                                        struct cfg80211_fils_discovery *fd)
{
        struct nlattr *tb[NL80211_FILS_DISCOVERY_ATTR_MAX + 1];
        int ret;

        if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_FILS_DISCOVERY))
                return -EINVAL;

        ret = nla_parse_nested(tb, NL80211_FILS_DISCOVERY_ATTR_MAX, attrs,
                               NULL, NULL);
        if (ret)
                return ret;

        if (!tb[NL80211_FILS_DISCOVERY_ATTR_INT_MIN] &&
            !tb[NL80211_FILS_DISCOVERY_ATTR_INT_MAX] &&
            !tb[NL80211_FILS_DISCOVERY_ATTR_TMPL]) {
                fd->update = true;
                return 0;
        }

        if (!tb[NL80211_FILS_DISCOVERY_ATTR_INT_MIN] ||
            !tb[NL80211_FILS_DISCOVERY_ATTR_INT_MAX] ||
            !tb[NL80211_FILS_DISCOVERY_ATTR_TMPL])
                return -EINVAL;

        fd->tmpl_len = nla_len(tb[NL80211_FILS_DISCOVERY_ATTR_TMPL]);
        fd->tmpl = nla_data(tb[NL80211_FILS_DISCOVERY_ATTR_TMPL]);
        fd->min_interval = nla_get_u32(tb[NL80211_FILS_DISCOVERY_ATTR_INT_MIN]);
        fd->max_interval = nla_get_u32(tb[NL80211_FILS_DISCOVERY_ATTR_INT_MAX]);
        fd->update = true;
        return 0;
}

static int
nl80211_parse_unsol_bcast_probe_resp(struct cfg80211_registered_device *rdev,
                                     struct nlattr *attrs,
                                     struct cfg80211_unsol_bcast_probe_resp *presp)
{
        struct nlattr *tb[NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_MAX + 1];
        int ret;

        if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_UNSOL_BCAST_PROBE_RESP))
                return -EINVAL;

        ret = nla_parse_nested(tb, NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_MAX,
                               attrs, NULL, NULL);
        if (ret)
                return ret;

        if (!tb[NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_INT] &&
            !tb[NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_TMPL]) {
                presp->update = true;
                return 0;
        }

        if (!tb[NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_INT] ||
            !tb[NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_TMPL])
                return -EINVAL;

        presp->tmpl = nla_data(tb[NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_TMPL]);
        presp->tmpl_len = nla_len(tb[NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_TMPL]);
        presp->interval = nla_get_u32(tb[NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_INT]);
        presp->update = true;
        return 0;
}

/*
 * Since the nl80211 API didn't include, from the beginning, attributes about
 * HT/VHT/... capabilities, we parse them out of the elements and check for
 * validity for use by drivers/mac80211.
 */
static int nl80211_calculate_ap_capabilities(struct genl_info *info,
                                             struct cfg80211_ap_settings *params)
{
        size_t ies_len = params->beacon.tail_len;
        const u8 *ies = params->beacon.tail;
        const struct element *cap;

        cap = cfg80211_find_elem(WLAN_EID_HT_CAPABILITY, ies, ies_len);
        if (cap) {
                if (cap->datalen < sizeof(*params->ht_cap)) {
                        GENL_SET_ERR_MSG(info, "bad HT capability in beacon");
                        return -EINVAL;
                }
                params->ht_cap = (void *)cap->data;
        }

        cap = cfg80211_find_elem(WLAN_EID_VHT_CAPABILITY, ies, ies_len);
        if (cap) {
                if (cap->datalen < sizeof(*params->vht_cap)) {
                        GENL_SET_ERR_MSG(info, "bad VHT capability in beacon");
                        return -EINVAL;
                }
                params->vht_cap = (void *)cap->data;
        }

        cap = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_CAPABILITY, ies, ies_len);
        if (cap) {
                if (cap->datalen < sizeof(*params->he_cap) + 1) {
                        GENL_SET_ERR_MSG(info, "bad HE capability in beacon");
                        return -EINVAL;
                }
                params->he_cap = (void *)(cap->data + 1);
        }

        cap = cfg80211_find_ext_elem(WLAN_EID_EXT_EHT_CAPABILITY, ies, ies_len);
        if (cap) {
                params->eht_cap = (void *)(cap->data + 1);
                if (!ieee80211_eht_capa_size_ok((const u8 *)params->he_cap,
                                                (const u8 *)params->eht_cap,
                                                cap->datalen - 1, true)) {
                        GENL_SET_ERR_MSG(info, "bad EHT capability in beacon");
                        return -EINVAL;
                }
        }

        if (!!params->he_cap != !!params->beacon.he_oper)
                return -EINVAL;

        if (!!params->eht_cap != !!params->beacon.eht_oper)
                return -EINVAL;

        return 0;
}

static bool nl80211_get_ap_channel(struct cfg80211_registered_device *rdev,
                                   struct cfg80211_ap_settings *params)
{
        struct wireless_dev *wdev;

        list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                if (wdev->iftype != NL80211_IFTYPE_AP &&
                    wdev->iftype != NL80211_IFTYPE_P2P_GO)
                        continue;

                if (!wdev->u.ap.preset_chandef.chan)
                        continue;

                params->chandef = wdev->u.ap.preset_chandef;
                return true;
        }

        return false;
}

static bool nl80211_valid_auth_type(struct cfg80211_registered_device *rdev,
                                    enum nl80211_auth_type auth_type,
                                    enum nl80211_commands cmd)
{
        if (auth_type > NL80211_AUTHTYPE_MAX)
                return false;

        switch (cmd) {
        case NL80211_CMD_AUTHENTICATE:
                if (!(rdev->wiphy.features & NL80211_FEATURE_SAE) &&
                    auth_type == NL80211_AUTHTYPE_SAE)
                        return false;
                if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_FILS_STA) &&
                    (auth_type == NL80211_AUTHTYPE_FILS_SK ||
                     auth_type == NL80211_AUTHTYPE_FILS_SK_PFS ||
                     auth_type == NL80211_AUTHTYPE_FILS_PK))
                        return false;
                if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_EPPKE) &&
                    auth_type == NL80211_AUTHTYPE_EPPKE)
                        return false;
                if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_IEEE8021X_AUTH) &&
                    auth_type == NL80211_AUTHTYPE_IEEE8021X)
                        return false;
                return true;
        case NL80211_CMD_CONNECT:
                if (!(rdev->wiphy.features & NL80211_FEATURE_SAE) &&
                    !wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_SAE_OFFLOAD) &&
                    auth_type == NL80211_AUTHTYPE_SAE)
                        return false;

                /* FILS with SK PFS or PK not supported yet */
                if (auth_type == NL80211_AUTHTYPE_FILS_SK_PFS ||
                    auth_type == NL80211_AUTHTYPE_FILS_PK)
                        return false;
                if (!wiphy_ext_feature_isset(
                            &rdev->wiphy,
                            NL80211_EXT_FEATURE_FILS_SK_OFFLOAD) &&
                    auth_type == NL80211_AUTHTYPE_FILS_SK)
                        return false;
                if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_EPPKE) &&
                    auth_type == NL80211_AUTHTYPE_EPPKE)
                        return false;
                if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_IEEE8021X_AUTH) &&
                    auth_type == NL80211_AUTHTYPE_IEEE8021X)
                        return false;
                return true;
        case NL80211_CMD_START_AP:
                if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_SAE_OFFLOAD_AP) &&
                    auth_type == NL80211_AUTHTYPE_SAE)
                        return false;
                /* FILS not supported yet */
                if (auth_type == NL80211_AUTHTYPE_FILS_SK ||
                    auth_type == NL80211_AUTHTYPE_FILS_SK_PFS ||
                    auth_type == NL80211_AUTHTYPE_FILS_PK)
                        return false;
                return true;
        default:
                return false;
        }
}

static void nl80211_send_ap_started(struct wireless_dev *wdev,
                                    unsigned int link_id)
{
        struct wiphy *wiphy = wdev->wiphy;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        struct sk_buff *msg;
        void *hdr;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return;

        hdr = nl80211hdr_put(msg, 0, 0, 0, NL80211_CMD_START_AP);
        if (!hdr)
                goto out;

        if (nla_put_u32(msg, NL80211_ATTR_WIPHY, rdev->wiphy_idx) ||
            nla_put_u32(msg, NL80211_ATTR_IFINDEX, wdev->netdev->ifindex) ||
            nla_put_u64_64bit(msg, NL80211_ATTR_WDEV, wdev_id(wdev),
                              NL80211_ATTR_PAD) ||
            (wdev->u.ap.ssid_len &&
             nla_put(msg, NL80211_ATTR_SSID, wdev->u.ap.ssid_len,
                     wdev->u.ap.ssid)) ||
            (wdev->valid_links &&
             nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_id)))
                goto out;

        genlmsg_end(msg, hdr);

        genlmsg_multicast_netns(&nl80211_fam, wiphy_net(wiphy), msg, 0,
                                NL80211_MCGRP_MLME, GFP_KERNEL);
        return;
out:
        nlmsg_free(msg);
}

static int
nl80211_parse_s1g_short_beacon(struct cfg80211_registered_device *rdev,
                               struct nlattr *attrs,
                               struct cfg80211_s1g_short_beacon *sb)
{
        struct nlattr *tb[NL80211_S1G_SHORT_BEACON_ATTR_MAX + 1];
        int ret;

        if (!rdev->wiphy.bands[NL80211_BAND_S1GHZ])
                return -EINVAL;

        ret = nla_parse_nested(tb, NL80211_S1G_SHORT_BEACON_ATTR_MAX, attrs,
                               NULL, NULL);
        if (ret)
                return ret;

        /* Short beacon tail is optional (i.e might only include the TIM) */
        if (!tb[NL80211_S1G_SHORT_BEACON_ATTR_HEAD])
                return -EINVAL;

        sb->short_head = nla_data(tb[NL80211_S1G_SHORT_BEACON_ATTR_HEAD]);
        sb->short_head_len = nla_len(tb[NL80211_S1G_SHORT_BEACON_ATTR_HEAD]);
        sb->short_tail_len = 0;

        if (tb[NL80211_S1G_SHORT_BEACON_ATTR_TAIL]) {
                sb->short_tail =
                        nla_data(tb[NL80211_S1G_SHORT_BEACON_ATTR_TAIL]);
                sb->short_tail_len =
                        nla_len(tb[NL80211_S1G_SHORT_BEACON_ATTR_TAIL]);
        }

        sb->update = true;
        return 0;
}

static int nl80211_check_npca(struct cfg80211_registered_device *rdev,
                              const struct cfg80211_chan_def *chandef,
                              enum nl80211_iftype iftype,
                              struct netlink_ext_ack *extack)
{
        const struct ieee80211_supported_band *sband;
        const struct ieee80211_sta_uhr_cap *uhr_cap;

        if (!chandef->npca_chan)
                return 0;

        sband = rdev->wiphy.bands[chandef->chan->band];
        uhr_cap = ieee80211_get_uhr_iftype_cap(sband, iftype);

        if (uhr_cap &&
            (uhr_cap->mac.mac_cap[0] & IEEE80211_UHR_MAC_CAP0_NPCA_SUPP))
                return 0;

        NL_SET_ERR_MSG(extack, "NPCA not supported");
        return -EINVAL;
}

static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct cfg80211_beaconing_check_config beacon_check = {};
        unsigned int link_id = nl80211_link_id(info->attrs);
        struct net_device *dev = info->user_ptr[1];
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_ap_settings *params;
        int err;

        if (dev->ieee80211_ptr->iftype != NL80211_IFTYPE_AP &&
            dev->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_GO)
                return -EOPNOTSUPP;

        if (!rdev->ops->start_ap)
                return -EOPNOTSUPP;

        if (wdev->links[link_id].cac_started)
                return -EBUSY;

        if (wdev->links[link_id].ap.beacon_interval)
                return -EALREADY;

        /* these are required for START_AP */
        if (!info->attrs[NL80211_ATTR_BEACON_INTERVAL] ||
            !info->attrs[NL80211_ATTR_DTIM_PERIOD] ||
            !info->attrs[NL80211_ATTR_BEACON_HEAD])
                return -EINVAL;

        if (info->attrs[NL80211_ATTR_SMPS_MODE] &&
            nla_get_u8(info->attrs[NL80211_ATTR_SMPS_MODE]) != NL80211_SMPS_OFF)
                return -EOPNOTSUPP;

        params = kzalloc_obj(*params);
        if (!params)
                return -ENOMEM;

        params->beacon_interval =
                nla_get_u32(info->attrs[NL80211_ATTR_BEACON_INTERVAL]);
        params->dtim_period =
                nla_get_u32(info->attrs[NL80211_ATTR_DTIM_PERIOD]);

        err = cfg80211_validate_beacon_int(rdev, dev->ieee80211_ptr->iftype,
                                           params->beacon_interval);
        if (err)
                goto out;

        /*
         * In theory, some of these attributes should be required here
         * but since they were not used when the command was originally
         * added, keep them optional for old user space programs to let
         * them continue to work with drivers that do not need the
         * additional information -- drivers must check!
         */
        if (info->attrs[NL80211_ATTR_SSID]) {
                params->ssid = nla_data(info->attrs[NL80211_ATTR_SSID]);
                params->ssid_len =
                        nla_len(info->attrs[NL80211_ATTR_SSID]);
                if (params->ssid_len == 0) {
                        err = -EINVAL;
                        goto out;
                }

                if (wdev->u.ap.ssid_len &&
                    (wdev->u.ap.ssid_len != params->ssid_len ||
                     memcmp(wdev->u.ap.ssid, params->ssid, params->ssid_len))) {
                        /* require identical SSID for MLO */
                        err = -EINVAL;
                        goto out;
                }
        } else if (wdev->valid_links) {
                /* require SSID for MLO */
                err = -EINVAL;
                goto out;
        }

        if (info->attrs[NL80211_ATTR_HIDDEN_SSID])
                params->hidden_ssid = nla_get_u32(
                        info->attrs[NL80211_ATTR_HIDDEN_SSID]);

        params->privacy = !!info->attrs[NL80211_ATTR_PRIVACY];

        if (info->attrs[NL80211_ATTR_AUTH_TYPE]) {
                params->auth_type = nla_get_u32(
                        info->attrs[NL80211_ATTR_AUTH_TYPE]);
                if (!nl80211_valid_auth_type(rdev, params->auth_type,
                                             NL80211_CMD_START_AP)) {
                        err = -EINVAL;
                        goto out;
                }
        } else
                params->auth_type = NL80211_AUTHTYPE_AUTOMATIC;

        err = nl80211_crypto_settings(rdev, info, &params->crypto,
                                      NL80211_MAX_NR_CIPHER_SUITES);
        if (err)
                goto out;

        if (info->attrs[NL80211_ATTR_INACTIVITY_TIMEOUT]) {
                if (!(rdev->wiphy.features & NL80211_FEATURE_INACTIVITY_TIMER)) {
                        err = -EOPNOTSUPP;
                        goto out;
                }
                params->inactivity_timeout = nla_get_u16(
                        info->attrs[NL80211_ATTR_INACTIVITY_TIMEOUT]);
        }

        if (info->attrs[NL80211_ATTR_P2P_CTWINDOW]) {
                if (dev->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_GO) {
                        err = -EINVAL;
                        goto out;
                }
                params->p2p_ctwindow =
                        nla_get_u8(info->attrs[NL80211_ATTR_P2P_CTWINDOW]);
                if (params->p2p_ctwindow != 0 &&
                    !(rdev->wiphy.features & NL80211_FEATURE_P2P_GO_CTWIN)) {
                        err = -EINVAL;
                        goto out;
                }
        }

        if (info->attrs[NL80211_ATTR_P2P_OPPPS]) {
                u8 tmp;

                if (dev->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_GO) {
                        err = -EINVAL;
                        goto out;
                }
                tmp = nla_get_u8(info->attrs[NL80211_ATTR_P2P_OPPPS]);
                params->p2p_opp_ps = tmp;
                if (params->p2p_opp_ps != 0 &&
                    !(rdev->wiphy.features & NL80211_FEATURE_P2P_GO_OPPPS)) {
                        err = -EINVAL;
                        goto out;
                }
        }

        if (info->attrs[NL80211_ATTR_WIPHY_FREQ]) {
                err = nl80211_parse_chandef(rdev, info->extack, info->attrs,
                                            &params->chandef, true);
                if (err)
                        goto out;
        } else if (wdev->valid_links) {
                /* with MLD need to specify the channel configuration */
                err = -EINVAL;
                goto out;
        } else if (wdev->u.ap.preset_chandef.chan) {
                params->chandef = wdev->u.ap.preset_chandef;
        } else if (!nl80211_get_ap_channel(rdev, params)) {
                err = -EINVAL;
                goto out;
        }

        err = nl80211_parse_beacon(rdev, info->attrs, &params->beacon,
                                   params->chandef.chan, info->extack);
        if (err)
                goto out;

        err = nl80211_check_npca(rdev, &params->chandef, wdev->iftype,
                                 info->extack);
        if (err)
                goto out;

        beacon_check.iftype = wdev->iftype;
        beacon_check.relax = true;
        beacon_check.reg_power =
                cfg80211_get_6ghz_power_type(params->beacon.tail,
                                             params->beacon.tail_len, 0);
        if (!cfg80211_reg_check_beaconing(&rdev->wiphy, &params->chandef,
                                          &beacon_check)) {
                err = -EINVAL;
                goto out;
        }

        if (info->attrs[NL80211_ATTR_TX_RATES]) {
                err = nl80211_parse_tx_bitrate_mask(info, info->attrs,
                                                    NL80211_ATTR_TX_RATES,
                                                    &params->beacon_rate,
                                                    dev, false, link_id);
                if (err)
                        goto out;

                err = validate_beacon_tx_rate(rdev, params->chandef.chan->band,
                                              &params->beacon_rate);
                if (err)
                        goto out;
        }

        params->pbss = nla_get_flag(info->attrs[NL80211_ATTR_PBSS]);
        if (params->pbss && !rdev->wiphy.bands[NL80211_BAND_60GHZ]) {
                err = -EOPNOTSUPP;
                goto out;
        }

        if (info->attrs[NL80211_ATTR_ACL_POLICY]) {
                params->acl = parse_acl_data(&rdev->wiphy, info);
                if (IS_ERR(params->acl)) {
                        err = PTR_ERR(params->acl);
                        params->acl = NULL;
                        goto out;
                }
        }

        params->twt_responder =
                    nla_get_flag(info->attrs[NL80211_ATTR_TWT_RESPONDER]);

        if (info->attrs[NL80211_ATTR_HE_OBSS_PD]) {
                err = nl80211_parse_he_obss_pd(
                                        info->attrs[NL80211_ATTR_HE_OBSS_PD],
                                        &params->he_obss_pd);
                if (err)
                        goto out;
        }

        if (info->attrs[NL80211_ATTR_FILS_DISCOVERY]) {
                err = nl80211_parse_fils_discovery(rdev,
                                                   info->attrs[NL80211_ATTR_FILS_DISCOVERY],
                                                   &params->fils_discovery);
                if (err)
                        goto out;
        }

        if (info->attrs[NL80211_ATTR_UNSOL_BCAST_PROBE_RESP]) {
                err = nl80211_parse_unsol_bcast_probe_resp(
                        rdev, info->attrs[NL80211_ATTR_UNSOL_BCAST_PROBE_RESP],
                        &params->unsol_bcast_probe_resp);
                if (err)
                        goto out;
        }

        if (info->attrs[NL80211_ATTR_MBSSID_CONFIG]) {
                err = nl80211_parse_mbssid_config(&rdev->wiphy, dev, link_id,
                                                  info->attrs[NL80211_ATTR_MBSSID_CONFIG],
                                                  &params->mbssid_config,
                                                  params->beacon.mbssid_ies ?
                                                        params->beacon.mbssid_ies->cnt :
                                                        0);
                if (err)
                        goto out;
        }

        if (!params->mbssid_config.ema && params->beacon.rnr_ies) {
                err = -EINVAL;
                goto out;
        }

        if (info->attrs[NL80211_ATTR_S1G_SHORT_BEACON]) {
                if (!info->attrs[NL80211_ATTR_S1G_LONG_BEACON_PERIOD]) {
                        err = -EINVAL;
                        goto out;
                }

                params->s1g_long_beacon_period = nla_get_u8(
                        info->attrs[NL80211_ATTR_S1G_LONG_BEACON_PERIOD]);

                err = nl80211_parse_s1g_short_beacon(
                        rdev, info->attrs[NL80211_ATTR_S1G_SHORT_BEACON],
                        &params->s1g_short_beacon);
                if (err)
                        goto out;
        }

        err = nl80211_calculate_ap_capabilities(info, params);
        if (err)
                goto out;

        if (info->attrs[NL80211_ATTR_AP_SETTINGS_FLAGS])
                params->flags = nla_get_u32(
                        info->attrs[NL80211_ATTR_AP_SETTINGS_FLAGS]);
        else if (info->attrs[NL80211_ATTR_EXTERNAL_AUTH_SUPPORT])
                params->flags |= NL80211_AP_SETTINGS_EXTERNAL_AUTH_SUPPORT;

        if (wdev->conn_owner_nlportid &&
            info->attrs[NL80211_ATTR_SOCKET_OWNER] &&
            wdev->conn_owner_nlportid != info->snd_portid) {
                err = -EINVAL;
                goto out;
        }

        /* FIXME: validate MLO/link-id against driver capabilities */

        err = rdev_start_ap(rdev, dev, params);
        if (!err) {
                wdev->links[link_id].ap.beacon_interval = params->beacon_interval;
                wdev->links[link_id].ap.chandef = params->chandef;
                wdev->u.ap.ssid_len = params->ssid_len;
                memcpy(wdev->u.ap.ssid, params->ssid,
                       params->ssid_len);

                if (info->attrs[NL80211_ATTR_SOCKET_OWNER])
                        wdev->conn_owner_nlportid = info->snd_portid;

                nl80211_send_ap_started(wdev, link_id);
        }
out:
        kfree(params->acl);
        kfree(params->beacon.mbssid_ies);
        if (params->mbssid_config.tx_wdev &&
            params->mbssid_config.tx_wdev->netdev &&
            params->mbssid_config.tx_wdev->netdev != dev)
                dev_put(params->mbssid_config.tx_wdev->netdev);
        kfree(params->beacon.rnr_ies);
        kfree(params);

        return err;
}

static int nl80211_set_beacon(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct cfg80211_beaconing_check_config beacon_check = {};
        unsigned int link_id = nl80211_link_id(info->attrs);
        struct net_device *dev = info->user_ptr[1];
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_ap_update *params;
        struct nlattr *attr;
        int err;

        if (dev->ieee80211_ptr->iftype != NL80211_IFTYPE_AP &&
            dev->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_GO)
                return -EOPNOTSUPP;

        if (!rdev->ops->change_beacon)
                return -EOPNOTSUPP;

        if (!wdev->links[link_id].ap.beacon_interval)
                return -EINVAL;

        params = kzalloc_obj(*params);
        if (!params)
                return -ENOMEM;

        err = nl80211_parse_beacon(rdev, info->attrs, &params->beacon,
                                   wdev->links[link_id].ap.chandef.chan,
                                   info->extack);
        if (err)
                goto out;

        /* recheck beaconing is permitted with possibly changed power type */
        beacon_check.iftype = wdev->iftype;
        beacon_check.relax = true;
        beacon_check.reg_power =
                cfg80211_get_6ghz_power_type(params->beacon.tail,
                                             params->beacon.tail_len, 0);
        if (!cfg80211_reg_check_beaconing(&rdev->wiphy,
                                          &wdev->links[link_id].ap.chandef,
                                          &beacon_check)) {
                err = -EINVAL;
                goto out;
        }

        attr = info->attrs[NL80211_ATTR_FILS_DISCOVERY];
        if (attr) {
                err = nl80211_parse_fils_discovery(rdev, attr,
                                                   &params->fils_discovery);
                if (err)
                        goto out;
        }

        attr = info->attrs[NL80211_ATTR_UNSOL_BCAST_PROBE_RESP];
        if (attr) {
                err = nl80211_parse_unsol_bcast_probe_resp(rdev, attr,
                                                           &params->unsol_bcast_probe_resp);
                if (err)
                        goto out;
        }

        attr = info->attrs[NL80211_ATTR_S1G_SHORT_BEACON];
        if (attr) {
                err = nl80211_parse_s1g_short_beacon(rdev, attr,
                                                     &params->s1g_short_beacon);
                if (err)
                        goto out;
        }

        err = rdev_change_beacon(rdev, dev, params);

out:
        kfree(params->beacon.mbssid_ies);
        kfree(params->beacon.rnr_ies);
        kfree(params);
        return err;
}

static int nl80211_stop_ap(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        unsigned int link_id = nl80211_link_id(info->attrs);
        struct net_device *dev = info->user_ptr[1];

        return cfg80211_stop_ap(rdev, dev, link_id, false);
}

static const struct nla_policy sta_flags_policy[NL80211_STA_FLAG_MAX + 1] = {
        [NL80211_STA_FLAG_AUTHORIZED] = { .type = NLA_FLAG },
        [NL80211_STA_FLAG_SHORT_PREAMBLE] = { .type = NLA_FLAG },
        [NL80211_STA_FLAG_WME] = { .type = NLA_FLAG },
        [NL80211_STA_FLAG_MFP] = { .type = NLA_FLAG },
        [NL80211_STA_FLAG_AUTHENTICATED] = { .type = NLA_FLAG },
        [NL80211_STA_FLAG_TDLS_PEER] = { .type = NLA_FLAG },
};

static int parse_station_flags(struct genl_info *info,
                               enum nl80211_iftype iftype,
                               struct station_parameters *params)
{
        struct nlattr *flags[NL80211_STA_FLAG_MAX + 1];
        struct nlattr *nla;
        int flag;

        /*
         * Try parsing the new attribute first so userspace
         * can specify both for older kernels.
         */
        nla = info->attrs[NL80211_ATTR_STA_FLAGS2];
        if (nla) {
                struct nl80211_sta_flag_update *sta_flags;

                sta_flags = nla_data(nla);
                params->sta_flags_mask = sta_flags->mask;
                params->sta_flags_set = sta_flags->set;
                params->sta_flags_set &= params->sta_flags_mask;
                if ((params->sta_flags_mask |
                     params->sta_flags_set) & BIT(__NL80211_STA_FLAG_INVALID))
                        return -EINVAL;

                if ((iftype == NL80211_IFTYPE_NAN ||
                     iftype == NL80211_IFTYPE_NAN_DATA) &&
                    params->sta_flags_mask &
                    ~(BIT(NL80211_STA_FLAG_AUTHENTICATED) |
                      BIT(NL80211_STA_FLAG_ASSOCIATED) |
                      BIT(NL80211_STA_FLAG_AUTHORIZED) |
                      BIT(NL80211_STA_FLAG_MFP)))
                                return -EINVAL;

                /* WME is always used in NAN */
                if (iftype == NL80211_IFTYPE_NAN_DATA) {
                        /* but don't let userspace control it */
                        if (params->sta_flags_mask & BIT(NL80211_STA_FLAG_WME))
                                return -EINVAL;

                        params->sta_flags_mask |= BIT(NL80211_STA_FLAG_WME);
                        params->sta_flags_set |= BIT(NL80211_STA_FLAG_WME);
                }

                return 0;
        }

        /* if present, parse the old attribute */

        nla = info->attrs[NL80211_ATTR_STA_FLAGS];
        if (!nla)
                return 0;

        if (nla_parse_nested_deprecated(flags, NL80211_STA_FLAG_MAX, nla, sta_flags_policy, info->extack))
                return -EINVAL;

        /*
         * Only allow certain flags for interface types so that
         * other attributes are silently ignored. Remember that
         * this is backward compatibility code with old userspace
         * and shouldn't be hit in other cases anyway.
         */
        switch (iftype) {
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_AP_VLAN:
        case NL80211_IFTYPE_P2P_GO:
                params->sta_flags_mask = BIT(NL80211_STA_FLAG_AUTHORIZED) |
                                         BIT(NL80211_STA_FLAG_SHORT_PREAMBLE) |
                                         BIT(NL80211_STA_FLAG_WME) |
                                         BIT(NL80211_STA_FLAG_MFP);
                break;
        case NL80211_IFTYPE_P2P_CLIENT:
        case NL80211_IFTYPE_STATION:
                params->sta_flags_mask = BIT(NL80211_STA_FLAG_AUTHORIZED) |
                                         BIT(NL80211_STA_FLAG_TDLS_PEER);
                break;
        case NL80211_IFTYPE_MESH_POINT:
                params->sta_flags_mask = BIT(NL80211_STA_FLAG_AUTHENTICATED) |
                                         BIT(NL80211_STA_FLAG_MFP) |
                                         BIT(NL80211_STA_FLAG_AUTHORIZED);
                break;
        default:
                return -EINVAL;
        }

        for (flag = 1; flag <= NL80211_STA_FLAG_MAX; flag++) {
                if (flags[flag]) {
                        params->sta_flags_set |= (1<<flag);

                        /* no longer support new API additions in old API */
                        if (flag > NL80211_STA_FLAG_MAX_OLD_API)
                                return -EINVAL;
                }
        }

        return 0;
}

bool nl80211_put_sta_rate(struct sk_buff *msg, struct rate_info *info, int attr)
{
        struct nlattr *rate;
        u32 bitrate;
        u16 bitrate_compat;
        enum nl80211_rate_info rate_flg;

        rate = nla_nest_start_noflag(msg, attr);
        if (!rate)
                return false;

        /* cfg80211_calculate_bitrate will return 0 for mcs >= 32 */
        bitrate = cfg80211_calculate_bitrate(info);
        /* report 16-bit bitrate only if we can */
        bitrate_compat = bitrate < (1UL << 16) ? bitrate : 0;
        if (bitrate > 0 &&
            nla_put_u32(msg, NL80211_RATE_INFO_BITRATE32, bitrate))
                return false;
        if (bitrate_compat > 0 &&
            nla_put_u16(msg, NL80211_RATE_INFO_BITRATE, bitrate_compat))
                return false;

        switch (info->bw) {
        case RATE_INFO_BW_1:
                rate_flg = NL80211_RATE_INFO_1_MHZ_WIDTH;
                break;
        case RATE_INFO_BW_2:
                rate_flg = NL80211_RATE_INFO_2_MHZ_WIDTH;
                break;
        case RATE_INFO_BW_4:
                rate_flg = NL80211_RATE_INFO_4_MHZ_WIDTH;
                break;
        case RATE_INFO_BW_5:
                rate_flg = NL80211_RATE_INFO_5_MHZ_WIDTH;
                break;
        case RATE_INFO_BW_8:
                rate_flg = NL80211_RATE_INFO_8_MHZ_WIDTH;
                break;
        case RATE_INFO_BW_10:
                rate_flg = NL80211_RATE_INFO_10_MHZ_WIDTH;
                break;
        case RATE_INFO_BW_16:
                rate_flg = NL80211_RATE_INFO_16_MHZ_WIDTH;
                break;
        default:
                WARN_ON(1);
                fallthrough;
        case RATE_INFO_BW_20:
                rate_flg = 0;
                break;
        case RATE_INFO_BW_40:
                rate_flg = NL80211_RATE_INFO_40_MHZ_WIDTH;
                break;
        case RATE_INFO_BW_80:
                rate_flg = NL80211_RATE_INFO_80_MHZ_WIDTH;
                break;
        case RATE_INFO_BW_160:
                rate_flg = NL80211_RATE_INFO_160_MHZ_WIDTH;
                break;
        case RATE_INFO_BW_HE_RU:
                rate_flg = 0;
                WARN_ON(!(info->flags & RATE_INFO_FLAGS_HE_MCS));
                break;
        case RATE_INFO_BW_320:
                rate_flg = NL80211_RATE_INFO_320_MHZ_WIDTH;
                break;
        case RATE_INFO_BW_EHT_RU:
                rate_flg = 0;
                WARN_ON(!(info->flags & RATE_INFO_FLAGS_EHT_MCS) &&
                        !(info->flags & RATE_INFO_FLAGS_UHR_MCS));
                break;
        }

        if (rate_flg && nla_put_flag(msg, rate_flg))
                return false;

        if (info->flags & RATE_INFO_FLAGS_MCS) {
                if (nla_put_u8(msg, NL80211_RATE_INFO_MCS, info->mcs))
                        return false;
                if (info->flags & RATE_INFO_FLAGS_SHORT_GI &&
                    nla_put_flag(msg, NL80211_RATE_INFO_SHORT_GI))
                        return false;
        } else if (info->flags & RATE_INFO_FLAGS_VHT_MCS) {
                if (nla_put_u8(msg, NL80211_RATE_INFO_VHT_MCS, info->mcs))
                        return false;
                if (nla_put_u8(msg, NL80211_RATE_INFO_VHT_NSS, info->nss))
                        return false;
                if (info->flags & RATE_INFO_FLAGS_SHORT_GI &&
                    nla_put_flag(msg, NL80211_RATE_INFO_SHORT_GI))
                        return false;
        } else if (info->flags & RATE_INFO_FLAGS_HE_MCS) {
                if (nla_put_u8(msg, NL80211_RATE_INFO_HE_MCS, info->mcs))
                        return false;
                if (nla_put_u8(msg, NL80211_RATE_INFO_HE_NSS, info->nss))
                        return false;
                if (nla_put_u8(msg, NL80211_RATE_INFO_HE_GI, info->he_gi))
                        return false;
                if (nla_put_u8(msg, NL80211_RATE_INFO_HE_DCM, info->he_dcm))
                        return false;
                if (info->bw == RATE_INFO_BW_HE_RU &&
                    nla_put_u8(msg, NL80211_RATE_INFO_HE_RU_ALLOC,
                               info->he_ru_alloc))
                        return false;
        } else if (info->flags & RATE_INFO_FLAGS_S1G_MCS) {
                if (nla_put_u8(msg, NL80211_RATE_INFO_S1G_MCS, info->mcs))
                        return false;
                if (nla_put_u8(msg, NL80211_RATE_INFO_S1G_NSS, info->nss))
                        return false;
                if (info->flags & RATE_INFO_FLAGS_SHORT_GI &&
                    nla_put_flag(msg, NL80211_RATE_INFO_SHORT_GI))
                        return false;
        } else if (info->flags & RATE_INFO_FLAGS_EHT_MCS) {
                if (nla_put_u8(msg, NL80211_RATE_INFO_EHT_MCS, info->mcs))
                        return false;
                if (nla_put_u8(msg, NL80211_RATE_INFO_EHT_NSS, info->nss))
                        return false;
                if (nla_put_u8(msg, NL80211_RATE_INFO_EHT_GI, info->eht_gi))
                        return false;
                if (info->bw == RATE_INFO_BW_EHT_RU &&
                    nla_put_u8(msg, NL80211_RATE_INFO_EHT_RU_ALLOC,
                               info->eht_ru_alloc))
                        return false;
        } else if (info->flags & RATE_INFO_FLAGS_UHR_MCS) {
                if (nla_put_u8(msg, NL80211_RATE_INFO_UHR_MCS, info->mcs))
                        return false;
                if (nla_put_u8(msg, NL80211_RATE_INFO_EHT_NSS, info->nss))
                        return false;
                if (nla_put_u8(msg, NL80211_RATE_INFO_EHT_GI, info->eht_gi))
                        return false;
                if (info->bw == RATE_INFO_BW_EHT_RU &&
                    nla_put_u8(msg, NL80211_RATE_INFO_EHT_RU_ALLOC,
                               info->eht_ru_alloc))
                        return false;
                if (info->flags & RATE_INFO_FLAGS_UHR_ELR_MCS &&
                    nla_put_flag(msg, NL80211_RATE_INFO_UHR_ELR))
                        return false;
                if (info->flags & RATE_INFO_FLAGS_UHR_IM &&
                    nla_put_flag(msg, NL80211_RATE_INFO_UHR_IM))
                        return false;
        }

        nla_nest_end(msg, rate);
        return true;
}

static bool nl80211_put_signal(struct sk_buff *msg, u8 mask, s8 *signal,
                               int id)
{
        void *attr;
        int i = 0;

        if (!mask)
                return true;

        attr = nla_nest_start_noflag(msg, id);
        if (!attr)
                return false;

        for (i = 0; i < IEEE80211_MAX_CHAINS; i++) {
                if (!(mask & BIT(i)))
                        continue;

                if (nla_put_u8(msg, i, signal[i]))
                        return false;
        }

        nla_nest_end(msg, attr);

        return true;
}

static int nl80211_fill_link_station(struct sk_buff *msg,
                                     struct cfg80211_registered_device *rdev,
                                     struct link_station_info *link_sinfo)
{
        struct nlattr *bss_param, *link_sinfoattr;

#define PUT_LINK_SINFO(attr, memb, type) do {                                \
        BUILD_BUG_ON(sizeof(type) == sizeof(u64));                        \
        if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_ ## attr) &&        \
            nla_put_ ## type(msg, NL80211_STA_INFO_ ## attr,                \
                             link_sinfo->memb))                                \
                goto nla_put_failure;                                        \
        } while (0)
#define PUT_LINK_SINFO_U64(attr, memb) do {                                \
        if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_ ## attr) &&        \
            nla_put_u64_64bit(msg, NL80211_STA_INFO_ ## attr,                \
                              link_sinfo->memb, NL80211_STA_INFO_PAD))        \
                goto nla_put_failure;                                        \
        } while (0)

        link_sinfoattr = nla_nest_start(msg, NL80211_ATTR_STA_INFO);
        if (!link_sinfoattr)
                goto nla_put_failure;

        PUT_LINK_SINFO(INACTIVE_TIME, inactive_time, u32);

        if (link_sinfo->filled & (BIT_ULL(NL80211_STA_INFO_RX_BYTES) |
                             BIT_ULL(NL80211_STA_INFO_RX_BYTES64)) &&
            nla_put_u32(msg, NL80211_STA_INFO_RX_BYTES,
                        (u32)link_sinfo->rx_bytes))
                goto nla_put_failure;

        if (link_sinfo->filled & (BIT_ULL(NL80211_STA_INFO_TX_BYTES) |
                             BIT_ULL(NL80211_STA_INFO_TX_BYTES64)) &&
            nla_put_u32(msg, NL80211_STA_INFO_TX_BYTES,
                        (u32)link_sinfo->tx_bytes))
                goto nla_put_failure;

        PUT_LINK_SINFO_U64(RX_BYTES64, rx_bytes);
        PUT_LINK_SINFO_U64(TX_BYTES64, tx_bytes);
        PUT_LINK_SINFO_U64(RX_DURATION, rx_duration);
        PUT_LINK_SINFO_U64(TX_DURATION, tx_duration);

        if (wiphy_ext_feature_isset(&rdev->wiphy,
                                    NL80211_EXT_FEATURE_AIRTIME_FAIRNESS))
                PUT_LINK_SINFO(AIRTIME_WEIGHT, airtime_weight, u16);

        switch (rdev->wiphy.signal_type) {
        case CFG80211_SIGNAL_TYPE_MBM:
                PUT_LINK_SINFO(SIGNAL, signal, u8);
                PUT_LINK_SINFO(SIGNAL_AVG, signal_avg, u8);
                break;
        default:
                break;
        }
        if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL)) {
                if (!nl80211_put_signal(msg, link_sinfo->chains,
                                        link_sinfo->chain_signal,
                                        NL80211_STA_INFO_CHAIN_SIGNAL))
                        goto nla_put_failure;
        }
        if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL_AVG)) {
                if (!nl80211_put_signal(msg, link_sinfo->chains,
                                        link_sinfo->chain_signal_avg,
                                        NL80211_STA_INFO_CHAIN_SIGNAL_AVG))
                        goto nla_put_failure;
        }
        if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_BITRATE)) {
                if (!nl80211_put_sta_rate(msg, &link_sinfo->txrate,
                                          NL80211_STA_INFO_TX_BITRATE))
                        goto nla_put_failure;
        }
        if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_RX_BITRATE)) {
                if (!nl80211_put_sta_rate(msg, &link_sinfo->rxrate,
                                          NL80211_STA_INFO_RX_BITRATE))
                        goto nla_put_failure;
        }

        PUT_LINK_SINFO(RX_PACKETS, rx_packets, u32);
        PUT_LINK_SINFO(TX_PACKETS, tx_packets, u32);
        PUT_LINK_SINFO(TX_RETRIES, tx_retries, u32);
        PUT_LINK_SINFO(TX_FAILED, tx_failed, u32);
        PUT_LINK_SINFO(EXPECTED_THROUGHPUT, expected_throughput, u32);
        PUT_LINK_SINFO(BEACON_LOSS, beacon_loss_count, u32);

        if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_BSS_PARAM)) {
                bss_param = nla_nest_start(msg,
                                           NL80211_STA_INFO_BSS_PARAM);
                if (!bss_param)
                        goto nla_put_failure;

                if (((link_sinfo->bss_param.flags &
                      BSS_PARAM_FLAGS_CTS_PROT) &&
                     nla_put_flag(msg, NL80211_STA_BSS_PARAM_CTS_PROT)) ||
                    ((link_sinfo->bss_param.flags &
                      BSS_PARAM_FLAGS_SHORT_PREAMBLE) &&
                     nla_put_flag(msg,
                                  NL80211_STA_BSS_PARAM_SHORT_PREAMBLE)) ||
                    ((link_sinfo->bss_param.flags &
                      BSS_PARAM_FLAGS_SHORT_SLOT_TIME) &&
                     nla_put_flag(msg,
                                  NL80211_STA_BSS_PARAM_SHORT_SLOT_TIME)) ||
                    nla_put_u8(msg, NL80211_STA_BSS_PARAM_DTIM_PERIOD,
                               link_sinfo->bss_param.dtim_period) ||
                    nla_put_u16(msg, NL80211_STA_BSS_PARAM_BEACON_INTERVAL,
                                link_sinfo->bss_param.beacon_interval))
                        goto nla_put_failure;

                nla_nest_end(msg, bss_param);
        }

        PUT_LINK_SINFO_U64(RX_DROP_MISC, rx_dropped_misc);
        PUT_LINK_SINFO_U64(BEACON_RX, rx_beacon);
        PUT_LINK_SINFO(BEACON_SIGNAL_AVG, rx_beacon_signal_avg, u8);
        PUT_LINK_SINFO(RX_MPDUS, rx_mpdu_count, u32);
        PUT_LINK_SINFO(FCS_ERROR_COUNT, fcs_err_count, u32);
        if (wiphy_ext_feature_isset(&rdev->wiphy,
                                    NL80211_EXT_FEATURE_ACK_SIGNAL_SUPPORT)) {
                PUT_LINK_SINFO(ACK_SIGNAL, ack_signal, u8);
                PUT_LINK_SINFO(ACK_SIGNAL_AVG, avg_ack_signal, s8);
        }

#undef PUT_LINK_SINFO
#undef PUT_LINK_SINFO_U64

        if (link_sinfo->pertid) {
                struct nlattr *tidsattr;
                int tid;

                tidsattr = nla_nest_start(msg, NL80211_STA_INFO_TID_STATS);
                if (!tidsattr)
                        goto nla_put_failure;

                for (tid = 0; tid < IEEE80211_NUM_TIDS + 1; tid++) {
                        struct cfg80211_tid_stats *tidstats;
                        struct nlattr *tidattr;

                        tidstats = &link_sinfo->pertid[tid];

                        if (!tidstats->filled)
                                continue;

                        tidattr = nla_nest_start(msg, tid + 1);
                        if (!tidattr)
                                goto nla_put_failure;

#define PUT_TIDVAL_U64(attr, memb) do {                                        \
        if (tidstats->filled & BIT(NL80211_TID_STATS_ ## attr) &&        \
            nla_put_u64_64bit(msg, NL80211_TID_STATS_ ## attr,                \
                              tidstats->memb, NL80211_TID_STATS_PAD))        \
                goto nla_put_failure;                                        \
        } while (0)

                        PUT_TIDVAL_U64(RX_MSDU, rx_msdu);
                        PUT_TIDVAL_U64(TX_MSDU, tx_msdu);
                        PUT_TIDVAL_U64(TX_MSDU_RETRIES, tx_msdu_retries);
                        PUT_TIDVAL_U64(TX_MSDU_FAILED, tx_msdu_failed);

#undef PUT_TIDVAL_U64
                        if ((tidstats->filled &
                             BIT(NL80211_TID_STATS_TXQ_STATS)) &&
                            !nl80211_put_txq_stats(msg, &tidstats->txq_stats,
                                                   NL80211_TID_STATS_TXQ_STATS))
                                goto nla_put_failure;

                        nla_nest_end(msg, tidattr);
                }

                nla_nest_end(msg, tidsattr);
        }

        nla_nest_end(msg, link_sinfoattr);
        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static int nl80211_put_sta_info_common(struct sk_buff *msg,
                                       struct cfg80211_registered_device *rdev,
                                       struct station_info *sinfo)
{
        struct nlattr *sinfoattr, *bss_param;

        sinfoattr = nla_nest_start(msg, NL80211_ATTR_STA_INFO);
        if (!sinfoattr)
                return -EMSGSIZE;

#define PUT_SINFO(attr, memb, type) do {                                \
        BUILD_BUG_ON(sizeof(type) == sizeof(u64));                        \
        if (sinfo->filled & BIT_ULL(NL80211_STA_INFO_ ## attr) &&        \
            nla_put_ ## type(msg, NL80211_STA_INFO_ ## attr,                \
                             sinfo->memb))                                \
                goto nla_put_failure;                                        \
        } while (0)
#define PUT_SINFO_U64(attr, memb) do {                                        \
        if (sinfo->filled & BIT_ULL(NL80211_STA_INFO_ ## attr) &&        \
            nla_put_u64_64bit(msg, NL80211_STA_INFO_ ## attr,                \
                              sinfo->memb, NL80211_STA_INFO_PAD))        \
                goto nla_put_failure;                                        \
        } while (0)

        PUT_SINFO(CONNECTED_TIME, connected_time, u32);
        PUT_SINFO(INACTIVE_TIME, inactive_time, u32);
        PUT_SINFO_U64(ASSOC_AT_BOOTTIME, assoc_at);

        if (sinfo->filled & (BIT_ULL(NL80211_STA_INFO_RX_BYTES) |
                             BIT_ULL(NL80211_STA_INFO_RX_BYTES64)) &&
            nla_put_u32(msg, NL80211_STA_INFO_RX_BYTES,
                        (u32)sinfo->rx_bytes))
                goto nla_put_failure;

        if (sinfo->filled & (BIT_ULL(NL80211_STA_INFO_TX_BYTES) |
                             BIT_ULL(NL80211_STA_INFO_TX_BYTES64)) &&
            nla_put_u32(msg, NL80211_STA_INFO_TX_BYTES,
                        (u32)sinfo->tx_bytes))
                goto nla_put_failure;

        PUT_SINFO_U64(RX_BYTES64, rx_bytes);
        PUT_SINFO_U64(TX_BYTES64, tx_bytes);
        PUT_SINFO_U64(RX_DURATION, rx_duration);
        PUT_SINFO_U64(TX_DURATION, tx_duration);

        if (wiphy_ext_feature_isset(&rdev->wiphy,
                                    NL80211_EXT_FEATURE_AIRTIME_FAIRNESS))
                PUT_SINFO(AIRTIME_WEIGHT, airtime_weight, u16);

        switch (rdev->wiphy.signal_type) {
        case CFG80211_SIGNAL_TYPE_MBM:
                PUT_SINFO(SIGNAL, signal, u8);
                PUT_SINFO(SIGNAL_AVG, signal_avg, u8);
                break;
        default:
                break;
        }
        if (sinfo->filled & BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL)) {
                if (!nl80211_put_signal(msg, sinfo->chains,
                                        sinfo->chain_signal,
                                        NL80211_STA_INFO_CHAIN_SIGNAL))
                        goto nla_put_failure;
        }
        if (sinfo->filled & BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL_AVG)) {
                if (!nl80211_put_signal(msg, sinfo->chains,
                                        sinfo->chain_signal_avg,
                                        NL80211_STA_INFO_CHAIN_SIGNAL_AVG))
                        goto nla_put_failure;
        }
        if (sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_BITRATE)) {
                if (!nl80211_put_sta_rate(msg, &sinfo->txrate,
                                          NL80211_STA_INFO_TX_BITRATE))
                        goto nla_put_failure;
        }
        if (sinfo->filled & BIT_ULL(NL80211_STA_INFO_RX_BITRATE)) {
                if (!nl80211_put_sta_rate(msg, &sinfo->rxrate,
                                          NL80211_STA_INFO_RX_BITRATE))
                        goto nla_put_failure;
        }

        PUT_SINFO(RX_PACKETS, rx_packets, u32);
        PUT_SINFO(TX_PACKETS, tx_packets, u32);
        PUT_SINFO(TX_RETRIES, tx_retries, u32);
        PUT_SINFO(TX_FAILED, tx_failed, u32);
        PUT_SINFO(EXPECTED_THROUGHPUT, expected_throughput, u32);
        PUT_SINFO(BEACON_LOSS, beacon_loss_count, u32);

        PUT_SINFO(LLID, llid, u16);
        PUT_SINFO(PLID, plid, u16);
        PUT_SINFO(PLINK_STATE, plink_state, u8);
        PUT_SINFO(AIRTIME_LINK_METRIC, airtime_link_metric, u32);
        PUT_SINFO(LOCAL_PM, local_pm, u32);
        PUT_SINFO(PEER_PM, peer_pm, u32);
        PUT_SINFO(NONPEER_PM, nonpeer_pm, u32);
        PUT_SINFO(CONNECTED_TO_GATE, connected_to_gate, u8);
        PUT_SINFO(CONNECTED_TO_AS, connected_to_as, u8);
        PUT_SINFO_U64(T_OFFSET, t_offset);

        if (sinfo->filled & BIT_ULL(NL80211_STA_INFO_BSS_PARAM)) {
                bss_param = nla_nest_start(msg, NL80211_STA_INFO_BSS_PARAM);
                if (!bss_param)
                        goto nla_put_failure;

                if (((sinfo->bss_param.flags & BSS_PARAM_FLAGS_CTS_PROT) &&
                     nla_put_flag(msg, NL80211_STA_BSS_PARAM_CTS_PROT)) ||
                    ((sinfo->bss_param.flags & BSS_PARAM_FLAGS_SHORT_PREAMBLE) &&
                     nla_put_flag(msg, NL80211_STA_BSS_PARAM_SHORT_PREAMBLE)) ||
                    ((sinfo->bss_param.flags & BSS_PARAM_FLAGS_SHORT_SLOT_TIME) &&
                     nla_put_flag(msg, NL80211_STA_BSS_PARAM_SHORT_SLOT_TIME)) ||
                    nla_put_u8(msg, NL80211_STA_BSS_PARAM_DTIM_PERIOD,
                               sinfo->bss_param.dtim_period) ||
                    nla_put_u16(msg, NL80211_STA_BSS_PARAM_BEACON_INTERVAL,
                                sinfo->bss_param.beacon_interval))
                        goto nla_put_failure;

                nla_nest_end(msg, bss_param);
        }
        if ((sinfo->filled & BIT_ULL(NL80211_STA_INFO_STA_FLAGS)) &&
            nla_put(msg, NL80211_STA_INFO_STA_FLAGS,
                    sizeof(struct nl80211_sta_flag_update),
                    &sinfo->sta_flags))
                goto nla_put_failure;

        PUT_SINFO_U64(RX_DROP_MISC, rx_dropped_misc);
        PUT_SINFO_U64(BEACON_RX, rx_beacon);
        PUT_SINFO(BEACON_SIGNAL_AVG, rx_beacon_signal_avg, u8);
        PUT_SINFO(RX_MPDUS, rx_mpdu_count, u32);
        PUT_SINFO(FCS_ERROR_COUNT, fcs_err_count, u32);
        if (wiphy_ext_feature_isset(&rdev->wiphy,
                                    NL80211_EXT_FEATURE_ACK_SIGNAL_SUPPORT)) {
                PUT_SINFO(ACK_SIGNAL, ack_signal, u8);
                PUT_SINFO(ACK_SIGNAL_AVG, avg_ack_signal, s8);
        }

#undef PUT_SINFO
#undef PUT_SINFO_U64

        if (sinfo->pertid) {
                struct nlattr *tidsattr;
                int tid;

                tidsattr = nla_nest_start(msg, NL80211_STA_INFO_TID_STATS);
                if (!tidsattr)
                        goto nla_put_failure;

                for (tid = 0; tid < IEEE80211_NUM_TIDS + 1; tid++) {
                        struct cfg80211_tid_stats *tidstats;
                        struct nlattr *tidattr;

                        tidstats = &sinfo->pertid[tid];

                        if (!tidstats->filled)
                                continue;

                        tidattr = nla_nest_start(msg, tid + 1);
                        if (!tidattr)
                                goto nla_put_failure;

#define PUT_TIDVAL_U64(attr, memb) do {                                        \
        if (tidstats->filled & BIT(NL80211_TID_STATS_ ## attr) &&        \
            nla_put_u64_64bit(msg, NL80211_TID_STATS_ ## attr,                \
                              tidstats->memb, NL80211_TID_STATS_PAD))        \
                goto nla_put_failure;                                        \
        } while (0)

                        PUT_TIDVAL_U64(RX_MSDU, rx_msdu);
                        PUT_TIDVAL_U64(TX_MSDU, tx_msdu);
                        PUT_TIDVAL_U64(TX_MSDU_RETRIES, tx_msdu_retries);
                        PUT_TIDVAL_U64(TX_MSDU_FAILED, tx_msdu_failed);

#undef PUT_TIDVAL_U64
                        if ((tidstats->filled &
                             BIT(NL80211_TID_STATS_TXQ_STATS)) &&
                            !nl80211_put_txq_stats(msg, &tidstats->txq_stats,
                                                   NL80211_TID_STATS_TXQ_STATS))
                                goto nla_put_failure;

                        nla_nest_end(msg, tidattr);
                }

                nla_nest_end(msg, tidsattr);
        }

        nla_nest_end(msg, sinfoattr);
        return 0;

nla_put_failure:
        nla_nest_cancel(msg, sinfoattr);
        return -EMSGSIZE;
}

static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid,
                                u32 seq, int flags,
                                struct cfg80211_registered_device *rdev,
                                struct wireless_dev *wdev,
                                const u8 *mac_addr, struct station_info *sinfo)
{
        void *hdr;

        hdr = nl80211hdr_put(msg, portid, seq, flags, cmd);
        if (!hdr) {
                cfg80211_sinfo_release_content(sinfo);
                return -1;
        }

        if ((wdev->netdev &&
             nla_put_u32(msg, NL80211_ATTR_IFINDEX, wdev->netdev->ifindex)) ||
            nla_put_u64_64bit(msg, NL80211_ATTR_WDEV, wdev_id(wdev),
                              NL80211_ATTR_PAD) ||
            nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, mac_addr) ||
            nla_put_u32(msg, NL80211_ATTR_GENERATION, sinfo->generation))
                goto nla_put_failure;

        if (nl80211_put_sta_info_common(msg, rdev, sinfo))
                goto nla_put_failure;

        if (sinfo->assoc_req_ies_len &&
            nla_put(msg, NL80211_ATTR_IE, sinfo->assoc_req_ies_len,
                    sinfo->assoc_req_ies))
                goto nla_put_failure;

        if (sinfo->assoc_resp_ies_len &&
            nla_put(msg, NL80211_ATTR_RESP_IE, sinfo->assoc_resp_ies_len,
                    sinfo->assoc_resp_ies))
                goto nla_put_failure;

        if (sinfo->mlo_params_valid) {
                if (nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID,
                               sinfo->assoc_link_id))
                        goto nla_put_failure;

                if (!is_zero_ether_addr(sinfo->mld_addr) &&
                    nla_put(msg, NL80211_ATTR_MLD_ADDR, ETH_ALEN,
                            sinfo->mld_addr))
                        goto nla_put_failure;
        }

        cfg80211_sinfo_release_content(sinfo);
        genlmsg_end(msg, hdr);
        return 0;

nla_put_failure:
        cfg80211_sinfo_release_content(sinfo);
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

static void cfg80211_sta_set_mld_sinfo(struct station_info *sinfo)
{
        struct link_station_info *link_sinfo;
        int link_id, init = 0;
        u32 link_inactive_time;

        sinfo->signal = -99;

        for_each_valid_link(sinfo, link_id) {
                link_sinfo = sinfo->links[link_id];
                if (!link_sinfo)
                        continue;

                if ((link_sinfo->filled &
                     BIT_ULL(NL80211_STA_INFO_TX_PACKETS))) {
                        sinfo->tx_packets += link_sinfo->tx_packets;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_PACKETS);
                }

                if ((link_sinfo->filled &
                     BIT_ULL(NL80211_STA_INFO_RX_PACKETS))) {
                        sinfo->rx_packets += link_sinfo->rx_packets;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS);
                }

                if (link_sinfo->filled &
                    (BIT_ULL(NL80211_STA_INFO_TX_BYTES) |
                     BIT_ULL(NL80211_STA_INFO_TX_BYTES64))) {
                        sinfo->tx_bytes += link_sinfo->tx_bytes;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES);
                }

                if (link_sinfo->filled &
                    (BIT_ULL(NL80211_STA_INFO_RX_BYTES) |
                     BIT_ULL(NL80211_STA_INFO_TX_BYTES64))) {
                        sinfo->rx_bytes += link_sinfo->rx_bytes;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES);
                }

                if (link_sinfo->filled &
                    BIT_ULL(NL80211_STA_INFO_TX_RETRIES)) {
                        sinfo->tx_retries += link_sinfo->tx_retries;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES);
                }

                if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_FAILED)) {
                        sinfo->tx_failed += link_sinfo->tx_failed;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
                }

                if (link_sinfo->filled &
                    BIT_ULL(NL80211_STA_INFO_RX_DROP_MISC)) {
                        sinfo->rx_dropped_misc += link_sinfo->rx_dropped_misc;
                        sinfo->filled |=
                                BIT_ULL(NL80211_STA_INFO_RX_DROP_MISC);
                }

                if (link_sinfo->filled &
                    BIT_ULL(NL80211_STA_INFO_BEACON_LOSS)) {
                        sinfo->beacon_loss_count +=
                                link_sinfo->beacon_loss_count;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_BEACON_LOSS);
                }

                if (link_sinfo->filled &
                    BIT_ULL(NL80211_STA_INFO_EXPECTED_THROUGHPUT)) {
                        sinfo->expected_throughput +=
                                link_sinfo->expected_throughput;
                        sinfo->filled |=
                                BIT_ULL(NL80211_STA_INFO_EXPECTED_THROUGHPUT);
                }

                if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_RX_MPDUS)) {
                        sinfo->rx_mpdu_count += link_sinfo->rx_mpdu_count;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_MPDUS);
                }

                if (link_sinfo->filled &
                    BIT_ULL(NL80211_STA_INFO_FCS_ERROR_COUNT)) {
                        sinfo->fcs_err_count += link_sinfo->fcs_err_count;
                        sinfo->filled |=
                                BIT_ULL(NL80211_STA_INFO_FCS_ERROR_COUNT);
                }

                if (link_sinfo->filled &
                    BIT_ULL(NL80211_STA_INFO_BEACON_RX)) {
                        sinfo->rx_beacon += link_sinfo->rx_beacon;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_BEACON_RX);
                }

                /* Update MLO signal, signal_avg as best among links */
                if ((link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_SIGNAL)) &&
                    link_sinfo->signal > sinfo->signal) {
                        sinfo->signal = link_sinfo->signal;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL);
                }

                if ((link_sinfo->filled &
                        BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG)) &&
                    link_sinfo->signal_avg > sinfo->signal_avg) {
                        sinfo->signal_avg = link_sinfo->signal_avg;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG);
                }

                /* Update MLO inactive_time, bss_param based on least
                 * value for corresponding field of link.
                 */
                if ((link_sinfo->filled &
                     BIT_ULL(NL80211_STA_INFO_INACTIVE_TIME)) &&
                    (!init ||
                     link_inactive_time > link_sinfo->inactive_time)) {
                        link_inactive_time = link_sinfo->inactive_time;
                        sinfo->inactive_time = link_sinfo->inactive_time;
                        sinfo->filled |= NL80211_STA_INFO_INACTIVE_TIME;
                }

                if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_BSS_PARAM) &&
                    (!init ||
                     sinfo->bss_param.dtim_period >
                      link_sinfo->bss_param.dtim_period)) {
                        sinfo->bss_param.dtim_period =
                                link_sinfo->bss_param.dtim_period;
                        sinfo->filled |= NL80211_STA_BSS_PARAM_DTIM_PERIOD;
                        sinfo->bss_param.beacon_interval =
                                link_sinfo->bss_param.beacon_interval;
                        sinfo->filled |= NL80211_STA_BSS_PARAM_BEACON_INTERVAL;
                }

                /* Update MLO rates as per last updated link rate */
                if ((link_sinfo->filled &
                     BIT_ULL(NL80211_STA_INFO_TX_BITRATE)) &&
                    (!init ||
                     link_inactive_time > link_sinfo->inactive_time)) {
                        sinfo->txrate = link_sinfo->txrate;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
                }
                if ((link_sinfo->filled &
                     BIT_ULL(NL80211_STA_INFO_RX_BITRATE)) &&
                    (!init ||
                     link_inactive_time > link_sinfo->inactive_time)) {
                        sinfo->rxrate = link_sinfo->rxrate;
                        sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BITRATE);
                }

                if (link_sinfo->filled &
                    BIT_ULL(NL80211_STA_INFO_TX_DURATION) &&
                    (!init ||
                     link_inactive_time > link_sinfo->inactive_time)) {
                        sinfo->tx_duration += link_sinfo->tx_duration;
                        sinfo->filled |=
                                BIT_ULL(NL80211_STA_INFO_TX_DURATION);
                }
                if (link_sinfo->filled &
                    BIT_ULL(NL80211_STA_INFO_RX_DURATION) &&
                    (!init ||
                     link_inactive_time > link_sinfo->inactive_time)) {
                        sinfo->rx_duration += link_sinfo->rx_duration;
                        sinfo->filled |=
                                BIT_ULL(NL80211_STA_INFO_RX_DURATION);
                }
                init++;

                /* pertid stats accumulate for rx/tx fields */
                if (sinfo->pertid) {
                        sinfo->pertid->rx_msdu +=
                                link_sinfo->pertid->rx_msdu;
                        sinfo->pertid->tx_msdu +=
                                link_sinfo->pertid->tx_msdu;
                        sinfo->pertid->tx_msdu_retries +=
                                link_sinfo->pertid->tx_msdu_retries;
                        sinfo->pertid->tx_msdu_failed +=
                                link_sinfo->pertid->tx_msdu_failed;

                        sinfo->pertid->filled |=
                                BIT(NL80211_TID_STATS_RX_MSDU) |
                                BIT(NL80211_TID_STATS_TX_MSDU) |
                                BIT(NL80211_TID_STATS_TX_MSDU_RETRIES) |
                                BIT(NL80211_TID_STATS_TX_MSDU_FAILED);
                }
        }

        /* Reset sinfo->filled bits to exclude fields which don't make
         * much sense at the MLO level.
         */
        sinfo->filled &= ~BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL);
        sinfo->filled &= ~BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL_AVG);
}

enum nl80211_dump_station_phase {
        NL80211_DUMP_STA_PHASE_AGGREGATED = 0,
        NL80211_DUMP_STA_PHASE_PER_LINK   = 1,
};

struct nl80211_dump_station_ctx {
        int sta_idx;
        int link_idx;
        enum nl80211_dump_station_phase phase;
        bool dump_link_stats;
        bool filter_mac;
        u8 filter_mac_addr[ETH_ALEN];
        u8 mac_addr[ETH_ALEN];
        struct station_info sinfo;
};

static int nl80211_put_link_station_payload(struct sk_buff *msg,
                                            struct cfg80211_registered_device *rdev,
                                            struct station_info *sinfo,
                                            int link_idx)
{
        struct link_station_info *link_sinfo = sinfo->links[link_idx];
        struct nlattr *links, *link;

        if (WARN_ON_ONCE(!link_sinfo))
                return -ENOENT;

        if (!is_valid_ether_addr(link_sinfo->addr))
                return -EADDRNOTAVAIL;

        links = nla_nest_start(msg, NL80211_ATTR_MLO_LINKS);
        if (!links)
                return -EMSGSIZE;

        link = nla_nest_start(msg, link_idx + 1);
        if (!link)
                goto nla_put_failure;

        if (nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_idx) ||
            nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, link_sinfo->addr))
                goto nla_put_failure;

        if (nl80211_fill_link_station(msg, rdev, link_sinfo))
                goto nla_put_failure;

        nla_nest_end(msg, link);
        nla_nest_end(msg, links);
        return 0;

nla_put_failure:
        nla_nest_cancel(msg, links);
        return -EMSGSIZE;
}

static int nl80211_dump_station(struct sk_buff *skb,
                                struct netlink_callback *cb)
{
        struct cfg80211_registered_device *rdev;
        struct wireless_dev *wdev;
        struct nl80211_dump_station_ctx *ctx = (void *)cb->args[2];
        struct nlattr **attrbuf __free(kfree) = NULL;
        int err;

        if (!ctx) {
                attrbuf = kzalloc_objs(*attrbuf, NUM_NL80211_ATTR);
                if (!attrbuf)
                        return -ENOMEM;
        }

        err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, attrbuf);
        if (err)
                return err;
        /* nl80211_prepare_wdev_dump acquired it in the successful case */
        __acquire(&rdev->wiphy.mtx);

        if (!ctx) {
                ctx = kzalloc_obj(*ctx);
                if (!ctx) {
                        err = -ENOMEM;
                        goto out_err;
                }
                cb->args[2] = (long)ctx;
                ctx->phase = NL80211_DUMP_STA_PHASE_AGGREGATED;
                ctx->dump_link_stats =
                        !!attrbuf[NL80211_ATTR_STA_DUMP_LINK_STATS];
                if (attrbuf[NL80211_ATTR_MAC]) {
                        const u8 *mac = nla_data(attrbuf[NL80211_ATTR_MAC]);

                        if (!is_valid_ether_addr(mac)) {
                                kfree(ctx);
                                cb->args[2] = 0;
                                err = -EINVAL;
                                goto out_err;
                        }
                        ctx->filter_mac = true;
                        memcpy(ctx->filter_mac_addr, mac, ETH_ALEN);
                }
        }

        if (!wdev->netdev && wdev->iftype != NL80211_IFTYPE_NAN) {
                err = -EINVAL;
                goto out_err;
        }

        if (ctx->filter_mac) {
                if (!rdev->ops->get_station) {
                        err = -EOPNOTSUPP;
                        goto out_err;
                }
        } else if (!rdev->ops->dump_station) {
                err = -EOPNOTSUPP;
                goto out_err;
        }

        while (true) {
                void *hdr;
                int ret;

                /* AGGREGATED phase: fetch sinfo from driver once per station */
                if (ctx->phase == NL80211_DUMP_STA_PHASE_AGGREGATED) {
                        memset(&ctx->sinfo, 0, sizeof(ctx->sinfo));
                        for (int i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
                                ctx->sinfo.links[i] =
                                        kzalloc_obj(*ctx->sinfo.links[0]);
                                if (!ctx->sinfo.links[i]) {
                                        err = -ENOMEM;
                                        goto out_err_release;
                                }
                        }

                        if (ctx->filter_mac) {
                                if (ctx->sta_idx > 0) {
                                        err = skb->len;
                                        goto out_err_release;
                                }
                                err = rdev_get_station(rdev, wdev,
                                                       ctx->filter_mac_addr,
                                                       &ctx->sinfo);
                                if (!err)
                                        memcpy(ctx->mac_addr,
                                               ctx->filter_mac_addr, ETH_ALEN);
                        } else {
                                err = rdev_dump_station(rdev, wdev, ctx->sta_idx,
                                                        ctx->mac_addr,
                                                        &ctx->sinfo);
                        }
                        if (err == -ENOENT) {
                                err = skb->len;
                                goto out_err_release;
                        }
                        if (err)
                                goto out_err_release;

                        if (ctx->sinfo.valid_links)
                                cfg80211_sta_set_mld_sinfo(&ctx->sinfo);
                } else {
                        /* PER_LINK phase: advance to next valid link */
                        while (ctx->link_idx < IEEE80211_MLD_MAX_NUM_LINKS &&
                               !(ctx->sinfo.valid_links & BIT(ctx->link_idx)))
                                ctx->link_idx++;

                        if (ctx->link_idx >= IEEE80211_MLD_MAX_NUM_LINKS) {
                                cfg80211_sinfo_release_content(&ctx->sinfo);
                                ctx->sta_idx++;
                                ctx->phase = NL80211_DUMP_STA_PHASE_AGGREGATED;
                                continue;
                        }
                }

                /* Build common header for both phases */
                hdr = nl80211hdr_put(skb, NETLINK_CB(cb->skb).portid,
                                     cb->nlh->nlmsg_seq, NLM_F_MULTI,
                                     NL80211_CMD_NEW_STATION);
                if (!hdr) {
                        err = skb->len;
                        if (ctx->phase == NL80211_DUMP_STA_PHASE_PER_LINK)
                                goto out_err;
                        goto out_err_release;
                }

                if ((wdev->netdev &&
                     nla_put_u32(skb, NL80211_ATTR_IFINDEX,
                                 wdev->netdev->ifindex)) ||
                    nla_put_u64_64bit(skb, NL80211_ATTR_WDEV,
                                      wdev_id(wdev), NL80211_ATTR_PAD) ||
                    nla_put(skb, NL80211_ATTR_MAC, ETH_ALEN, ctx->mac_addr) ||
                    nla_put_u32(skb, NL80211_ATTR_GENERATION,
                                ctx->sinfo.generation)) {
                        genlmsg_cancel(skb, hdr);
                        err = skb->len;
                        if (ctx->phase == NL80211_DUMP_STA_PHASE_PER_LINK)
                                goto out_err;
                        goto out_err_release;
                }

                switch (ctx->phase) {
                case NL80211_DUMP_STA_PHASE_AGGREGATED:
                        ret = nl80211_put_sta_info_common(skb, rdev, &ctx->sinfo);
                        if (ret) {
                                genlmsg_cancel(skb, hdr);
                                err = ret;
                                goto out_err_release;
                        }
                        genlmsg_end(skb, hdr);

                        if (ctx->dump_link_stats && ctx->sinfo.valid_links) {
                                ctx->phase = NL80211_DUMP_STA_PHASE_PER_LINK;
                                ctx->link_idx = 0;
                        } else {
                                cfg80211_sinfo_release_content(&ctx->sinfo);
                                ctx->sta_idx++;
                        }
                        break;

                case NL80211_DUMP_STA_PHASE_PER_LINK:
                        ret = nl80211_put_link_station_payload(skb, rdev,
                                                               &ctx->sinfo,
                                                               ctx->link_idx);
                        if (ret == -EMSGSIZE) {
                                genlmsg_cancel(skb, hdr);
                                err = skb->len;
                                goto out_err;
                        }
                        if (ret) {
                                /* skip invalid link, do not abort the dump */
                                genlmsg_cancel(skb, hdr);
                                ctx->link_idx++;
                                continue;
                        }
                        genlmsg_end(skb, hdr);
                        ctx->link_idx++;
                        break;
                }
        }

out_err_release:
        cfg80211_sinfo_release_content(&ctx->sinfo);
        memset(&ctx->sinfo, 0, sizeof(ctx->sinfo));
out_err:
        wiphy_unlock(&rdev->wiphy);

        return err;
}

static int nl80211_dump_station_done(struct netlink_callback *cb)
{
        struct nl80211_dump_station_ctx *ctx = (void *)cb->args[2];

        if (ctx) {
                cfg80211_sinfo_release_content(&ctx->sinfo);
                kfree(ctx);
        }
        return 0;
}

static int nl80211_get_station(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct wireless_dev *wdev = info->user_ptr[1];
        struct station_info sinfo;
        struct sk_buff *msg;
        u8 *mac_addr = NULL;
        int err, i;

        memset(&sinfo, 0, sizeof(sinfo));

        if (!wdev->netdev)
                return -EINVAL;

        if (!info->attrs[NL80211_ATTR_MAC])
                return -EINVAL;

        mac_addr = nla_data(info->attrs[NL80211_ATTR_MAC]);

        if (!rdev->ops->get_station)
                return -EOPNOTSUPP;

        for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
                sinfo.links[i] = kzalloc_obj(*sinfo.links[0]);
                if (!sinfo.links[i]) {
                        cfg80211_sinfo_release_content(&sinfo);
                        return -ENOMEM;
                }
        }

        err = rdev_get_station(rdev, wdev, mac_addr, &sinfo);
        if (err) {
                cfg80211_sinfo_release_content(&sinfo);
                return err;
        }

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg) {
                cfg80211_sinfo_release_content(&sinfo);
                return -ENOMEM;
        }

        if (sinfo.valid_links)
                cfg80211_sta_set_mld_sinfo(&sinfo);

        if (nl80211_send_station(msg, NL80211_CMD_NEW_STATION,
                                 info->snd_portid, info->snd_seq, 0,
                                 rdev, wdev, mac_addr, &sinfo) < 0) {
                nlmsg_free(msg);
                return -ENOBUFS;
        }

        return genlmsg_reply(msg, info);
}

int cfg80211_check_station_change(struct wiphy *wiphy,
                                  struct station_parameters *params,
                                  enum cfg80211_station_type statype)
{
        if (params->listen_interval != -1 &&
            statype != CFG80211_STA_AP_CLIENT_UNASSOC)
                return -EINVAL;

        if (params->support_p2p_ps != -1 &&
            statype != CFG80211_STA_AP_CLIENT_UNASSOC)
                return -EINVAL;

        if (params->aid &&
            !(params->sta_flags_set & BIT(NL80211_STA_FLAG_TDLS_PEER)) &&
            statype != CFG80211_STA_AP_CLIENT_UNASSOC)
                return -EINVAL;

        /* When you run into this, adjust the code below for the new flag */
        BUILD_BUG_ON(NL80211_STA_FLAG_MAX != 8);

        switch (statype) {
        case CFG80211_STA_MESH_PEER_KERNEL:
        case CFG80211_STA_MESH_PEER_USER:
                /*
                 * No ignoring the TDLS flag here -- the userspace mesh
                 * code doesn't have the bug of including TDLS in the
                 * mask everywhere.
                 */
                if (params->sta_flags_mask &
                                ~(BIT(NL80211_STA_FLAG_AUTHENTICATED) |
                                  BIT(NL80211_STA_FLAG_MFP) |
                                  BIT(NL80211_STA_FLAG_AUTHORIZED)))
                        return -EINVAL;
                break;
        case CFG80211_STA_TDLS_PEER_SETUP:
        case CFG80211_STA_TDLS_PEER_ACTIVE:
                if (!(params->sta_flags_set & BIT(NL80211_STA_FLAG_TDLS_PEER)))
                        return -EINVAL;
                /* ignore since it can't change */
                params->sta_flags_mask &= ~BIT(NL80211_STA_FLAG_TDLS_PEER);
                break;
        default:
                /* disallow mesh-specific things */
                if (params->plink_action != NL80211_PLINK_ACTION_NO_ACTION)
                        return -EINVAL;
                if (params->local_pm)
                        return -EINVAL;
                if (params->sta_modify_mask & STATION_PARAM_APPLY_PLINK_STATE)
                        return -EINVAL;
        }

        if (statype != CFG80211_STA_TDLS_PEER_SETUP &&
            statype != CFG80211_STA_TDLS_PEER_ACTIVE) {
                /* TDLS can't be set, ... */
                if (params->sta_flags_set & BIT(NL80211_STA_FLAG_TDLS_PEER))
                        return -EINVAL;
                /*
                 * ... but don't bother the driver with it. This works around
                 * a hostapd/wpa_supplicant issue -- it always includes the
                 * TLDS_PEER flag in the mask even for AP mode.
                 */
                params->sta_flags_mask &= ~BIT(NL80211_STA_FLAG_TDLS_PEER);
        }

        if (statype != CFG80211_STA_TDLS_PEER_SETUP &&
            statype != CFG80211_STA_AP_CLIENT_UNASSOC) {
                /* reject other things that can't change */
                if (params->sta_modify_mask & STATION_PARAM_APPLY_UAPSD)
                        return -EINVAL;
                if (params->sta_modify_mask & STATION_PARAM_APPLY_CAPABILITY)
                        return -EINVAL;
                if (params->link_sta_params.supported_rates)
                        return -EINVAL;
                if (statype != CFG80211_STA_NAN_MGMT &&
                    (params->link_sta_params.ht_capa ||
                     params->link_sta_params.vht_capa ||
                     params->link_sta_params.he_capa))
                        return -EINVAL;
                if (params->ext_capab || params->link_sta_params.eht_capa ||
                    params->link_sta_params.uhr_capa)
                        return -EINVAL;
                if (params->sta_flags_mask & BIT(NL80211_STA_FLAG_SPP_AMSDU))
                        return -EINVAL;
        }

        if (statype != CFG80211_STA_AP_CLIENT &&
            statype != CFG80211_STA_AP_CLIENT_UNASSOC) {
                if (params->vlan)
                        return -EINVAL;
        }

        /* Accept EMLSR capabilities only for AP client before association */
        if (statype != CFG80211_STA_AP_CLIENT_UNASSOC &&
            params->eml_cap_present)
                return -EINVAL;

        switch (statype) {
        case CFG80211_STA_AP_MLME_CLIENT:
                /* Use this only for authorizing/unauthorizing a station */
                if (!(params->sta_flags_mask & BIT(NL80211_STA_FLAG_AUTHORIZED)))
                        return -EOPNOTSUPP;
                break;
        case CFG80211_STA_AP_CLIENT:
        case CFG80211_STA_AP_CLIENT_UNASSOC:
                /* accept only the listed bits */
                if (params->sta_flags_mask &
                                ~(BIT(NL80211_STA_FLAG_AUTHORIZED) |
                                  BIT(NL80211_STA_FLAG_AUTHENTICATED) |
                                  BIT(NL80211_STA_FLAG_ASSOCIATED) |
                                  BIT(NL80211_STA_FLAG_SHORT_PREAMBLE) |
                                  BIT(NL80211_STA_FLAG_WME) |
                                  BIT(NL80211_STA_FLAG_MFP) |
                                  BIT(NL80211_STA_FLAG_SPP_AMSDU)))
                        return -EINVAL;

                /* but authenticated/associated only if driver handles it */
                if (!(wiphy->features & NL80211_FEATURE_FULL_AP_CLIENT_STATE) &&
                    params->sta_flags_mask &
                                (BIT(NL80211_STA_FLAG_AUTHENTICATED) |
                                 BIT(NL80211_STA_FLAG_ASSOCIATED)))
                        return -EINVAL;
                break;
        case CFG80211_STA_IBSS:
        case CFG80211_STA_AP_STA:
                /* reject any changes other than AUTHORIZED */
                if (params->sta_flags_mask & ~BIT(NL80211_STA_FLAG_AUTHORIZED))
                        return -EINVAL;
                break;
        case CFG80211_STA_TDLS_PEER_SETUP:
                /* reject any changes other than AUTHORIZED or WME */
                if (params->sta_flags_mask & ~(BIT(NL80211_STA_FLAG_AUTHORIZED) |
                                               BIT(NL80211_STA_FLAG_WME)))
                        return -EINVAL;
                /* force (at least) rates when authorizing */
                if (params->sta_flags_set & BIT(NL80211_STA_FLAG_AUTHORIZED) &&
                    !params->link_sta_params.supported_rates)
                        return -EINVAL;
                break;
        case CFG80211_STA_TDLS_PEER_ACTIVE:
                /* reject any changes */
                return -EINVAL;
        case CFG80211_STA_MESH_PEER_KERNEL:
                if (params->sta_modify_mask & STATION_PARAM_APPLY_PLINK_STATE)
                        return -EINVAL;
                break;
        case CFG80211_STA_MESH_PEER_USER:
                if (params->plink_action != NL80211_PLINK_ACTION_NO_ACTION &&
                    params->plink_action != NL80211_PLINK_ACTION_BLOCK)
                        return -EINVAL;
                break;
        case CFG80211_STA_NAN_MGMT:
                if (params->sta_flags_mask &
                    ~(BIT(NL80211_STA_FLAG_AUTHORIZED) |
                      BIT(NL80211_STA_FLAG_MFP)))
                        return -EINVAL;
                break;
        case CFG80211_STA_NAN_DATA:
                if (params->sta_flags_mask &
                    ~(BIT(NL80211_STA_FLAG_AUTHORIZED) |
                      BIT(NL80211_STA_FLAG_MFP) |
                      BIT(NL80211_STA_FLAG_WME)))
                        return -EINVAL;
                break;
        }

        /*
         * Older kernel versions ignored this attribute entirely, so don't
         * reject attempts to update it but mark it as unused instead so the
         * driver won't look at the data.
         */
        if (statype != CFG80211_STA_AP_CLIENT_UNASSOC &&
            statype != CFG80211_STA_TDLS_PEER_SETUP)
                params->link_sta_params.opmode_notif_used = false;

        return 0;
}
EXPORT_SYMBOL(cfg80211_check_station_change);

/*
 * Get vlan interface making sure it is running and on the right wiphy.
 */
static struct net_device *get_vlan(struct genl_info *info,
                                   struct cfg80211_registered_device *rdev)
{
        struct nlattr *vlanattr = info->attrs[NL80211_ATTR_STA_VLAN];
        struct net_device *v;
        int ret;

        if (!vlanattr)
                return NULL;

        v = dev_get_by_index(genl_info_net(info), nla_get_u32(vlanattr));
        if (!v)
                return ERR_PTR(-ENODEV);

        if (!v->ieee80211_ptr || v->ieee80211_ptr->wiphy != &rdev->wiphy) {
                ret = -EINVAL;
                goto error;
        }

        if (v->ieee80211_ptr->iftype != NL80211_IFTYPE_AP_VLAN &&
            v->ieee80211_ptr->iftype != NL80211_IFTYPE_AP &&
            v->ieee80211_ptr->iftype != NL80211_IFTYPE_P2P_GO) {
                ret = -EINVAL;
                goto error;
        }

        if (!netif_running(v)) {
                ret = -ENETDOWN;
                goto error;
        }

        return v;
 error:
        dev_put(v);
        return ERR_PTR(ret);
}

static int nl80211_parse_sta_wme(struct genl_info *info,
                                 struct station_parameters *params)
{
        struct nlattr *tb[NL80211_STA_WME_MAX + 1];
        struct nlattr *nla;
        int err;

        /* parse WME attributes if present */
        if (!info->attrs[NL80211_ATTR_STA_WME])
                return 0;

        nla = info->attrs[NL80211_ATTR_STA_WME];
        err = nla_parse_nested_deprecated(tb, NL80211_STA_WME_MAX, nla,
                                          nl80211_sta_wme_policy,
                                          info->extack);
        if (err)
                return err;

        if (tb[NL80211_STA_WME_UAPSD_QUEUES])
                params->uapsd_queues = nla_get_u8(
                        tb[NL80211_STA_WME_UAPSD_QUEUES]);
        if (params->uapsd_queues & ~IEEE80211_WMM_IE_STA_QOSINFO_AC_MASK)
                return -EINVAL;

        if (tb[NL80211_STA_WME_MAX_SP])
                params->max_sp = nla_get_u8(tb[NL80211_STA_WME_MAX_SP]);

        if (params->max_sp & ~IEEE80211_WMM_IE_STA_QOSINFO_SP_MASK)
                return -EINVAL;

        params->sta_modify_mask |= STATION_PARAM_APPLY_UAPSD;

        return 0;
}

static int nl80211_parse_sta_channel_info(struct genl_info *info,
                                      struct station_parameters *params)
{
        if (info->attrs[NL80211_ATTR_STA_SUPPORTED_CHANNELS]) {
                params->supported_channels =
                     nla_data(info->attrs[NL80211_ATTR_STA_SUPPORTED_CHANNELS]);
                params->supported_channels_len =
                     nla_len(info->attrs[NL80211_ATTR_STA_SUPPORTED_CHANNELS]);
                /*
                 * Need to include at least one (first channel, number of
                 * channels) tuple for each subband (checked in policy),
                 * and must have proper tuples for the rest of the data as well.
                 */
                if (params->supported_channels_len % 2)
                        return -EINVAL;
        }

        if (info->attrs[NL80211_ATTR_STA_SUPPORTED_OPER_CLASSES]) {
                params->supported_oper_classes =
                 nla_data(info->attrs[NL80211_ATTR_STA_SUPPORTED_OPER_CLASSES]);
                params->supported_oper_classes_len =
                  nla_len(info->attrs[NL80211_ATTR_STA_SUPPORTED_OPER_CLASSES]);
        }
        return 0;
}

static int nl80211_set_station_tdls(struct genl_info *info,
                                    struct station_parameters *params)
{
        int err;
        /* Dummy STA entry gets updated once the peer capabilities are known */
        if (info->attrs[NL80211_ATTR_PEER_AID])
                params->aid = nla_get_u16(info->attrs[NL80211_ATTR_PEER_AID]);
        if (info->attrs[NL80211_ATTR_HT_CAPABILITY])
                params->link_sta_params.ht_capa =
                        nla_data(info->attrs[NL80211_ATTR_HT_CAPABILITY]);
        if (info->attrs[NL80211_ATTR_VHT_CAPABILITY])
                params->link_sta_params.vht_capa =
                        nla_data(info->attrs[NL80211_ATTR_VHT_CAPABILITY]);
        if (info->attrs[NL80211_ATTR_HE_CAPABILITY]) {
                params->link_sta_params.he_capa =
                        nla_data(info->attrs[NL80211_ATTR_HE_CAPABILITY]);
                params->link_sta_params.he_capa_len =
                        nla_len(info->attrs[NL80211_ATTR_HE_CAPABILITY]);

                if (info->attrs[NL80211_ATTR_EHT_CAPABILITY]) {
                        params->link_sta_params.eht_capa =
                                nla_data(info->attrs[NL80211_ATTR_EHT_CAPABILITY]);
                        params->link_sta_params.eht_capa_len =
                                nla_len(info->attrs[NL80211_ATTR_EHT_CAPABILITY]);

                        if (!ieee80211_eht_capa_size_ok((const u8 *)params->link_sta_params.he_capa,
                                                        (const u8 *)params->link_sta_params.eht_capa,
                                                        params->link_sta_params.eht_capa_len,
                                                        false))
                                return -EINVAL;
                }
        }

        if (info->attrs[NL80211_ATTR_UHR_CAPABILITY]) {
                if (!params->link_sta_params.eht_capa)
                        return -EINVAL;

                params->link_sta_params.uhr_capa =
                        nla_data(info->attrs[NL80211_ATTR_UHR_CAPABILITY]);
                params->link_sta_params.uhr_capa_len =
                        nla_len(info->attrs[NL80211_ATTR_UHR_CAPABILITY]);
        }

        if (info->attrs[NL80211_ATTR_S1G_CAPABILITY])
                params->link_sta_params.s1g_capa =
                        nla_data(info->attrs[NL80211_ATTR_S1G_CAPABILITY]);

        err = nl80211_parse_sta_channel_info(info, params);
        if (err)
                return err;

        return nl80211_parse_sta_wme(info, params);
}

static int nl80211_parse_sta_txpower_setting(struct genl_info *info,
                                             struct sta_txpwr *txpwr,
                                             bool *txpwr_set)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        int idx;

        if (info->attrs[NL80211_ATTR_STA_TX_POWER_SETTING]) {
                if (!rdev->ops->set_tx_power ||
                    !wiphy_ext_feature_isset(&rdev->wiphy,
                                         NL80211_EXT_FEATURE_STA_TX_PWR))
                        return -EOPNOTSUPP;

                idx = NL80211_ATTR_STA_TX_POWER_SETTING;
                txpwr->type = nla_get_u8(info->attrs[idx]);

                if (txpwr->type == NL80211_TX_POWER_LIMITED) {
                        idx = NL80211_ATTR_STA_TX_POWER;

                        if (info->attrs[idx])
                                txpwr->power = nla_get_s16(info->attrs[idx]);
                        else
                                return -EINVAL;
                }

                *txpwr_set = true;
        } else {
                *txpwr_set = false;
        }

        return 0;
}

static int nl80211_set_station(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        struct wireless_dev *wdev = info->user_ptr[1];
        struct net_device *dev = wdev->netdev;
        struct station_parameters params;
        u8 *mac_addr;
        int err;

        memset(&params, 0, sizeof(params));

        if (!dev && wdev->iftype != NL80211_IFTYPE_NAN &&
            wdev->iftype != NL80211_IFTYPE_NAN_DATA)
                return -EINVAL;

        if (!rdev->ops->change_station)
                return -EOPNOTSUPP;

        /*
         * AID and listen_interval properties can be set only for unassociated
         * station. Include these parameters here and will check them in
         * cfg80211_check_station_change().
         */
        if (info->attrs[NL80211_ATTR_STA_AID])
                params.aid = nla_get_u16(info->attrs[NL80211_ATTR_STA_AID]);

        if (info->attrs[NL80211_ATTR_VLAN_ID])
                params.vlan_id = nla_get_u16(info->attrs[NL80211_ATTR_VLAN_ID]);

        if (info->attrs[NL80211_ATTR_STA_LISTEN_INTERVAL])
                params.listen_interval =
                     nla_get_u16(info->attrs[NL80211_ATTR_STA_LISTEN_INTERVAL]);
        else
                params.listen_interval = -1;

        if (info->attrs[NL80211_ATTR_STA_SUPPORT_P2P_PS])
                params.support_p2p_ps =
                        nla_get_u8(info->attrs[NL80211_ATTR_STA_SUPPORT_P2P_PS]);
        else
                params.support_p2p_ps = -1;

        if (!info->attrs[NL80211_ATTR_MAC])
                return -EINVAL;

        params.link_sta_params.link_id =
                nl80211_link_id_or_invalid(info->attrs);

        if (info->attrs[NL80211_ATTR_MLD_ADDR]) {
                /* If MLD_ADDR attribute is set then this is an MLD station
                 * and the MLD_ADDR attribute holds the MLD address and the
                 * MAC attribute holds for the LINK address.
                 * In that case, the link_id is also expected to be valid.
                 */
                if (params.link_sta_params.link_id < 0)
                        return -EINVAL;

                mac_addr = nla_data(info->attrs[NL80211_ATTR_MLD_ADDR]);
                params.link_sta_params.mld_mac = mac_addr;
                params.link_sta_params.link_mac =
                        nla_data(info->attrs[NL80211_ATTR_MAC]);
                if (!is_valid_ether_addr(params.link_sta_params.link_mac))
                        return -EINVAL;
        } else {
                mac_addr = nla_data(info->attrs[NL80211_ATTR_MAC]);
        }


        if (info->attrs[NL80211_ATTR_STA_SUPPORTED_RATES]) {
                params.link_sta_params.supported_rates =
                        nla_data(info->attrs[NL80211_ATTR_STA_SUPPORTED_RATES]);
                params.link_sta_params.supported_rates_len =
                        nla_len(info->attrs[NL80211_ATTR_STA_SUPPORTED_RATES]);
        }

        if (info->attrs[NL80211_ATTR_STA_CAPABILITY]) {
                params.capability =
                        nla_get_u16(info->attrs[NL80211_ATTR_STA_CAPABILITY]);
                params.sta_modify_mask |= STATION_PARAM_APPLY_CAPABILITY;
        }

        if (info->attrs[NL80211_ATTR_STA_EXT_CAPABILITY]) {
                params.ext_capab =
                        nla_data(info->attrs[NL80211_ATTR_STA_EXT_CAPABILITY]);
                params.ext_capab_len =
                        nla_len(info->attrs[NL80211_ATTR_STA_EXT_CAPABILITY]);
        }

        if (parse_station_flags(info, wdev->iftype, &params))
                return -EINVAL;

        if (info->attrs[NL80211_ATTR_STA_PLINK_ACTION])
                params.plink_action =
                        nla_get_u8(info->attrs[NL80211_ATTR_STA_PLINK_ACTION]);

        if (info->attrs[NL80211_ATTR_STA_PLINK_STATE]) {
                params.plink_state =
                        nla_get_u8(info->attrs[NL80211_ATTR_STA_PLINK_STATE]);
                if (info->attrs[NL80211_ATTR_MESH_PEER_AID])
                        params.peer_aid = nla_get_u16(
                                info->attrs[NL80211_ATTR_MESH_PEER_AID]);
                params.sta_modify_mask |= STATION_PARAM_APPLY_PLINK_STATE;
        }

        if (info->attrs[NL80211_ATTR_LOCAL_MESH_POWER_MODE])
                params.local_pm = nla_get_u32(
                        info->attrs[NL80211_ATTR_LOCAL_MESH_POWER_MODE]);

        if (info->attrs[NL80211_ATTR_OPMODE_NOTIF]) {
                params.link_sta_params.opmode_notif_used = true;
                params.link_sta_params.opmode_notif =
                        nla_get_u8(info->attrs[NL80211_ATTR_OPMODE_NOTIF]);
        }

        if (info->attrs[NL80211_ATTR_HE_6GHZ_CAPABILITY])
                params.link_sta_params.he_6ghz_capa =
                        nla_data(info->attrs[NL80211_ATTR_HE_6GHZ_CAPABILITY]);

        if (info->attrs[NL80211_ATTR_EML_CAPABILITY]) {
                params.eml_cap_present = true;
                params.eml_cap =
                        nla_get_u16(info->attrs[NL80211_ATTR_EML_CAPABILITY]);
        }

        if (info->attrs[NL80211_ATTR_AIRTIME_WEIGHT])
                params.airtime_weight =
                        nla_get_u16(info->attrs[NL80211_ATTR_AIRTIME_WEIGHT]);

        if (params.airtime_weight &&
            !wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_AIRTIME_FAIRNESS))
                return -EOPNOTSUPP;

        err = nl80211_parse_sta_txpower_setting(info,
                                                &params.link_sta_params.txpwr,
                                                &params.link_sta_params.txpwr_set);
        if (err)
                return err;

        /* Include parameters for TDLS peer (will check later) */
        err = nl80211_set_station_tdls(info, &params);
        if (err)
                return err;

        params.vlan = get_vlan(info, rdev);
        if (IS_ERR(params.vlan))
                return PTR_ERR(params.vlan);

        switch (wdev->iftype) {
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_AP_VLAN:
        case NL80211_IFTYPE_P2P_GO:
        case NL80211_IFTYPE_P2P_CLIENT:
        case NL80211_IFTYPE_STATION:
        case NL80211_IFTYPE_ADHOC:
        case NL80211_IFTYPE_MESH_POINT:
        case NL80211_IFTYPE_NAN:
        case NL80211_IFTYPE_NAN_DATA:
                break;
        default:
                err = -EOPNOTSUPP;
                goto out_put_vlan;
        }

        /* driver will call cfg80211_check_station_change() */
        err = rdev_change_station(rdev, wdev, mac_addr, &params);

 out_put_vlan:
        dev_put(params.vlan);

        return err;
}

static int nl80211_new_station(struct sk_buff *skb, struct genl_info *info)
{
        struct cfg80211_registered_device *rdev = info->user_ptr[0];
        int err;
        struct wireless_dev *wdev = info->user_ptr[1];
        struct net_device *dev = wdev->netdev;
        struct station_parameters params;
        u8 *mac_addr = NULL;
        u32 auth_assoc = BIT(NL80211_STA_FLAG_AUTHENTICATED) |
                         BIT(NL80211_STA_FLAG_ASSOCIATED);

        memset(&params, 0, sizeof(params));

        if (!dev && wdev->iftype != NL80211_IFTYPE_NAN)
                return -EINVAL;

        if (!rdev->ops->add_station)
                return -EOPNOTSUPP;

        if (!info->attrs[NL80211_ATTR_MAC])
                return -EINVAL;

        if (wdev->iftype == NL80211_IFTYPE_NAN ||
            wdev->iftype == NL80211_IFTYPE_NAN_DATA) {
                if (info->attrs[NL80211_ATTR_STA_SUPPORTED_RATES])
                        return -EINVAL;
                if (wdev->iftype == NL80211_IFTYPE_NAN_DATA) {
                        if (!info->attrs[NL80211_ATTR_NAN_NMI_MAC])
                                return -EINVAL;

                        /* Only NMI stations receive the HT/VHT/HE capabilities */
                        if (info->attrs[NL80211_ATTR_HT_CAPABILITY] ||
                            info->attrs[NL80211_ATTR_VHT_CAPABILITY] ||
                            info->attrs[NL80211_ATTR_HE_CAPABILITY])
                                return -EINVAL;
                }
        } else {
                if (!info->attrs[NL80211_ATTR_STA_LISTEN_INTERVAL])
                        return -EINVAL;

                if (!info->attrs[NL80211_ATTR_STA_SUPPORTED_RATES])
                        return -EINVAL;

                if (!info->attrs[NL80211_ATTR_STA_AID] &&
                    !info->attrs[NL80211_ATTR_PEER_AID])
                        return -EINVAL;
        }

        params.link_sta_params.link_id =
                nl80211_link_id_or_invalid(info->attrs);

        if (info->attrs[NL80211_ATTR_MLD_ADDR]) {
                mac_addr = nla_data(info->attrs[NL80211_ATTR_MLD_ADDR]);
                params.link_sta_params.mld_mac = mac_addr;
                params.link_sta_params.link_mac =
                        nla_data(info->attrs[NL80211_ATTR_MAC]);
                if (!is_valid_ether_addr(params.link_sta_params.link_mac))
                        return -EINVAL;
        } else {
                mac_addr = nla_data(info->attrs[NL80211_ATTR_MAC]);
        }

        if (info->attrs[NL80211_ATTR_STA_SUPPORTED_RATES]) {
                params.link_sta_params.supported_rates =
                        nla_data(info->attrs[NL80211_ATTR_STA_SUPPORTED_RATES]);
                params.link_sta_params.supported_rates_len =
                        nla_len(info->attrs[NL80211_ATTR_STA_SUPPORTED_RATES]);
        }

        if (info->attrs[NL80211_ATTR_STA_LISTEN_INTERVAL])
                params.listen_interval =
                        nla_get_u16(info->attrs[NL80211_ATTR_STA_LISTEN_INTERVAL]);

        if (info->attrs[NL80211_ATTR_VLAN_ID])
                params.vlan_id = nla_get_u16(info->attrs[NL80211_ATTR_VLAN_ID]);

        if (info->attrs[NL80211_ATTR_STA_SUPPORT_P2P_PS]) {
                params.support_p2p_ps =
                        nla_get_u8(info->attrs[NL80211_ATTR_STA_SUPPORT_P2P_PS]);
        } else {
                /*
                 * if not specified, assume it's supported for P2P GO interface,
                 * and is NOT supported for AP interface
                 */
                params.support_p2p_ps =
                        wdev->iftype == NL80211_IFTYPE_P2P_GO;
        }

        if (info->attrs[NL80211_ATTR_PEER_AID])
                params.aid = nla_get_u16(info->attrs[NL80211_ATTR_PEER_AID]);
        else if (info->attrs[NL80211_ATTR_STA_AID])
                params.aid = nla_get_u16(info->attrs[NL80211_ATTR_STA_AID]);

        if (info->attrs[NL80211_ATTR_STA_CAPABILITY]) {
                params.capability =
                        nla_get_u16(info->attrs[NL80211_ATTR_STA_CAPABILITY]);
                params.sta_modify_mask |= STATION_PARAM_APPLY_CAPABILITY;
        }

        if (info->attrs[NL80211_ATTR_STA_EXT_CAPABILITY]) {
                params.ext_capab =
                        nla_data(info->attrs[NL80211_ATTR_STA_EXT_CAPABILITY]);
                params.ext_capab_len =
                        nla_len(info->attrs[NL80211_ATTR_STA_EXT_CAPABILITY]);
        }

        if (info->attrs[NL80211_ATTR_HT_CAPABILITY])
                params.link_sta_params.ht_capa =
                        nla_data(info->attrs[NL80211_ATTR_HT_CAPABILITY]);

        if (info->attrs[NL80211_ATTR_VHT_CAPABILITY])
                params.link_sta_params.vht_capa =
                        nla_data(info->attrs[NL80211_ATTR_VHT_CAPABILITY]);

        if (info->attrs[NL80211_ATTR_HE_CAPABILITY]) {
                params.link_sta_params.he_capa =
                        nla_data(info->attrs[NL80211_ATTR_HE_CAPABILITY]);
                params.link_sta_params.he_capa_len =
                        nla_len(info->attrs[NL80211_ATTR_HE_CAPABILITY]);

                if (info->attrs[NL80211_ATTR_EHT_CAPABILITY]) {
                        params.link_sta_params.eht_capa =
                                nla_data(info->attrs[NL80211_ATTR_EHT_CAPABILITY]);
                        params.link_sta_params.eht_capa_len =
                                nla_len(info->attrs[NL80211_ATTR_EHT_CAPABILITY]);

                        if (!ieee80211_eht_capa_size_ok((const u8 *)params.link_sta_params.he_capa,
                                                        (const u8 *)params.link_sta_params.eht_capa,
                                                        params.link_sta_params.eht_capa_len,
                                                        false))
                                return -EINVAL;
                }
        }

        if (info->attrs[NL80211_ATTR_UHR_CAPABILITY]) {
                if (!params.link_sta_params.eht_capa)
                        return -EINVAL;

                params.link_sta_params.uhr_capa =
                        nla_data(info->attrs[NL80211_ATTR_UHR_CAPABILITY]);
                params.link_sta_params.uhr_capa_len =
                        nla_len(info->attrs[NL80211_ATTR_UHR_CAPABILITY]);
        }

        if (info->attrs[NL80211_ATTR_EML_CAPABILITY]) {
                params.eml_cap_present = true;
                params.eml_cap =
                        nla_get_u16(info->attrs[NL80211_ATTR_EML_CAPABILITY]);
        }

        if (info->attrs[NL80211_ATTR_HE_6GHZ_CAPABILITY])
                params.link_sta_params.he_6ghz_capa =
                        nla_data(info->attrs[NL80211_ATTR_HE_6GHZ_CAPABILITY]);

        if (info->attrs[NL80211_ATTR_S1G_CAPABILITY])
                params.link_sta_params.s1g_capa =
                        nla_data(info->attrs[NL80211_ATTR_S1G_CAPABILITY]);

        if (info->attrs[NL80211_ATTR_OPMODE_NOTIF]) {
                params.link_sta_params.opmode_notif_used = true;
                params.link_sta_params.opmode_notif =
                        nla_get_u8(info->attrs[NL80211_ATTR_OPMODE_NOTIF]);
        }

        if (info->attrs[NL80211_ATTR_STA_PLINK_ACTION])
                params.plink_action =
                        nla_get_u8(info->attrs[NL80211_ATTR_STA_PLINK_ACTION]);

        if (info->attrs[NL80211_ATTR_AIRTIME_WEIGHT])
                params.airtime_weight =
                        nla_get_u16(info->attrs[NL80211_ATTR_AIRTIME_WEIGHT]);

        if (params.airtime_weight &&
            !wiphy_ext_feature_isset(&rdev->wiphy,
                                     NL80211_EXT_FEATURE_AIRTIME_FAIRNESS))
                return -EOPNOTSUPP;

        err = nl80211_parse_sta_txpower_setting(info,
                                                &params.link_sta_params.txpwr,
                                                &params.link_sta_params.txpwr_set);
        if (err)
                return err;

        err = nl80211_parse_sta_channel_info(info, &params);
        if (err)
                return err;

        err = nl80211_parse_sta_wme(info, &params);
        if (err)
                return err;

        if (parse_station_flags(info, wdev->iftype, &params))
                return -EINVAL;

        /* HT/VHT requires QoS, but if we don't have that just ignore HT/VHT
         * as userspace might just pass through the capabilities from the IEs
         * directly, rather than enforcing this restriction and returning an
         * error in this case.
         */
        if (!(params.sta_flags_set & BIT(NL80211_STA_FLAG_WME))) {
                params.link_sta_params.ht_capa = NULL;
                params.link_sta_params.vht_capa = NULL;

                /* HE, EHT and UHR require WME */
                if (params.link_sta_params.he_capa_len ||
                    params.link_sta_params.he_6ghz_capa ||
                    params.link_sta_params.eht_capa_len ||
                    params.link_sta_params.uhr_capa_len)
                        return -EINVAL;
        }

        if (wdev->iftype == NL80211_IFTYPE_NAN ||
            wdev->iftype == NL80211_IFTYPE_NAN_DATA) {
                if (params.sta_modify_mask & STATION_PARAM_APPLY_UAPSD)
                        return -EINVAL;
                /* NAN NMI station must be added in associated or authorized state */
                if (!(params.sta_flags_set & (BIT(NL80211_STA_FLAG_ASSOCIATED) |
                                              BIT(NL80211_STA_FLAG_AUTHENTICATED))))
                        return -EINVAL;
        }

        /* Ensure that HT/VHT capabilities are not set for 6 GHz HE STA */
        if (params.link_sta_params.he_6ghz_capa &&
            (params.link_sta_params.ht_capa || params.link_sta_params.vht_capa))
                return -EINVAL;

        /* When you run into this, adjust the code below for the new flag */
        BUILD_BUG_ON(NL80211_STA_FLAG_MAX != 8);

        switch (wdev->iftype) {
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_AP_VLAN:
        case NL80211_IFTYPE_P2P_GO:
                /* ignore WME attributes if iface/sta is not capable */
                if (!(rdev->wiphy.flags & WIPHY_FLAG_AP_UAPSD) ||
                    !(params.sta_flags_set & BIT(NL80211_STA_FLAG_WME)))
                        params.sta_modify_mask &= ~STATION_PARAM_APPLY_UAPSD;

                /* TDLS peers cannot be added */
                if ((params.sta_flags_set & BIT(NL80211_STA_FLAG_TDLS_PEER)) ||
                    info->attrs[NL80211_ATTR_PEER_AID])
                        return -EINVAL;
                /* but don't bother the driver with it */
                params.sta_flags_mask &= ~BIT(NL80211_STA_FLAG_TDLS_PEER);

                /* allow authenticated/associated only if driver handles it */
                if (!(rdev->wiphy.features &
                                NL80211_FEATURE_FULL_AP_CLIENT_STATE) &&
                    params.sta_flags_mask & auth_assoc)
                        return -EINVAL;

                if (!wiphy_ext_feature_isset(&rdev->wiphy,
                                             NL80211_EXT_FEATURE_SPP_AMSDU_SUPPORT) &&
                    params.sta_flags_mask & BIT(NL80211_STA_FLAG_SPP_AMSDU))
                        return -EINVAL;

                /* Older userspace, or userspace wanting to be compatible with
                 * !NL80211_FEATURE_FULL_AP_CLIENT_STATE, will not set the auth
                 * and assoc flags in the mask, but assumes the station will be
                 * added as associated anyway since this was the required driver
                 * behaviour before NL80211_FEATURE_FULL_AP_CLIENT_STATE was
                 * introduced.
                 * In order to not bother drivers with this quirk in the API
                 * set the flags in both the mask and set for new stations in
                 * this case.
                 */
                if (!(params.sta_flags_mask & auth_assoc)) {
                        params.sta_flags_mask |= auth_assoc;
                        params.sta_flags_set |= auth_assoc;
                }